2024-05-20T22:03:26.692 INFO:root:teuthology version: 0.0.1.dev287+gb9e3da8 2024-05-20T22:03:26.693 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_adking@teuthology --verbose --name adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --archive /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783 -- /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783/orig.config.yaml 2024-05-20T22:03:26.729 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-20T22:03:26.812 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783 branch: wip-adk4-testing-2024-05-17-0821-squid description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7715783' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk4-testing-2024-05-17-0821-squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 8a373da3c102b503bf0a94126977cebf498afc8b ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 8a373da3c102b503bf0a94126977cebf498afc8b selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-adk4-testing-2024-05-17-0821-squid sha1: 8a373da3c102b503bf0a94126977cebf498afc8b owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 1485 sha1: 8a373da3c102b503bf0a94126977cebf498afc8b sleep_before_teardown: 0 subset: 1/18 suite: orch:cephadm suite_branch: wip-adk4-testing-2024-05-17-0821-squid suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_8a373da3c102b503bf0a94126977cebf498afc8b/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 8a373da3c102b503bf0a94126977cebf498afc8b targets: smithi006.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAhyEMhkFV16InQC5T5HfyjbntvJ1YdF9XdvwX1pAnB+g63oXYyh+igPxC+T6Lw09p4nBehqR75ZxNlPZhQ0IKA= smithi078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHKthEvMyFqh/QtJr8/HMImDe/c2hl/xxC0pWUq1nKdP7VFiic9f9Ti7CONZagvoKVb8JZnTkCBDQwxVKLrREpA= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: b9e3da87b1ec7920937cb4902b7e3fd9eda86b4a timestamp: 2024-05-20_18:23:59 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2698841 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-05-20T22:03:26.812 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_8a373da3c102b503bf0a94126977cebf498afc8b/qa; will attempt to use it 2024-05-20T22:03:26.813 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_8a373da3c102b503bf0a94126977cebf498afc8b/qa/tasks 2024-05-20T22:03:26.813 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-05-20T22:03:26.814 INFO:teuthology.task.internal:Checking packages... 2024-05-20T22:03:26.835 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '8a373da3c102b503bf0a94126977cebf498afc8b' 2024-05-20T22:03:26.835 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-05-20T22:03:26.835 INFO:teuthology.packaging:ref: None 2024-05-20T22:03:26.835 INFO:teuthology.packaging:tag: None 2024-05-20T22:03:26.835 INFO:teuthology.packaging:branch: wip-adk4-testing-2024-05-17-0821-squid 2024-05-20T22:03:26.835 INFO:teuthology.packaging:sha1: 8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:03:26.835 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk4-testing-2024-05-17-0821-squid 2024-05-20T22:03:26.971 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2695.g8a373da3 2024-05-20T22:03:26.972 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-05-20T22:03:26.979 INFO:teuthology.task.internal:no buildpackages task found 2024-05-20T22:03:26.979 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-05-20T22:03:27.003 INFO:teuthology.task.internal:Saving configuration 2024-05-20T22:03:27.017 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-05-20T22:03:27.074 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-05-20T22:03:27.100 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi006.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-20 21:58:27.059555', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAhyEMhkFV16InQC5T5HfyjbntvJ1YdF9XdvwX1pAnB+g63oXYyh+igPxC+T6Lw09p4nBehqR75ZxNlPZhQ0IKA='} 2024-05-20T22:03:27.118 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi078.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-05-20 21:58:27.060884', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHKthEvMyFqh/QtJr8/HMImDe/c2hl/xxC0pWUq1nKdP7VFiic9f9Ti7CONZagvoKVb8JZnTkCBDQwxVKLrREpA='} 2024-05-20T22:03:27.118 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-05-20T22:03:27.150 INFO:teuthology.task.internal:roles: ubuntu@smithi006.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-05-20T22:03:27.150 INFO:teuthology.task.internal:roles: ubuntu@smithi078.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-05-20T22:03:27.150 INFO:teuthology.run_tasks:Running task console_log... 2024-05-20T22:03:27.253 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb86273bac0>, signals=[15]) 2024-05-20T22:03:27.253 INFO:teuthology.run_tasks:Running task internal.connect... 2024-05-20T22:03:27.259 INFO:teuthology.task.internal:Opening connections... 2024-05-20T22:03:27.259 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:03:27.261 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:03:27.383 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi078.front.sepia.ceph.com 2024-05-20T22:03:27.384 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:03:27.461 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-05-20T22:03:27.467 DEBUG:teuthology.orchestra.run.smithi006:> uname -m 2024-05-20T22:03:27.485 INFO:teuthology.orchestra.run.smithi006.stdout:x86_64 2024-05-20T22:03:27.486 DEBUG:teuthology.orchestra.run.smithi006:> cat /etc/os-release 2024-05-20T22:03:27.543 INFO:teuthology.orchestra.run.smithi006.stdout:NAME="CentOS Stream" 2024-05-20T22:03:27.543 INFO:teuthology.orchestra.run.smithi006.stdout:VERSION="9" 2024-05-20T22:03:27.543 INFO:teuthology.orchestra.run.smithi006.stdout:ID="centos" 2024-05-20T22:03:27.543 INFO:teuthology.orchestra.run.smithi006.stdout:ID_LIKE="rhel fedora" 2024-05-20T22:03:27.543 INFO:teuthology.orchestra.run.smithi006.stdout:VERSION_ID="9" 2024-05-20T22:03:27.543 INFO:teuthology.orchestra.run.smithi006.stdout:PLATFORM_ID="platform:el9" 2024-05-20T22:03:27.543 INFO:teuthology.orchestra.run.smithi006.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-20T22:03:27.544 INFO:teuthology.orchestra.run.smithi006.stdout:ANSI_COLOR="0;31" 2024-05-20T22:03:27.544 INFO:teuthology.orchestra.run.smithi006.stdout:LOGO="fedora-logo-icon" 2024-05-20T22:03:27.544 INFO:teuthology.orchestra.run.smithi006.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-20T22:03:27.544 INFO:teuthology.orchestra.run.smithi006.stdout:HOME_URL="https://centos.org/" 2024-05-20T22:03:27.544 INFO:teuthology.orchestra.run.smithi006.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-20T22:03:27.544 INFO:teuthology.orchestra.run.smithi006.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-20T22:03:27.544 INFO:teuthology.orchestra.run.smithi006.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-20T22:03:27.545 INFO:teuthology.lock.ops:Updating smithi006.front.sepia.ceph.com on lock server 2024-05-20T22:03:27.570 DEBUG:teuthology.orchestra.run.smithi078:> uname -m 2024-05-20T22:03:27.587 INFO:teuthology.orchestra.run.smithi078.stdout:x86_64 2024-05-20T22:03:27.588 DEBUG:teuthology.orchestra.run.smithi078:> cat /etc/os-release 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:NAME="CentOS Stream" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION="9" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:ID="centos" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:ID_LIKE="rhel fedora" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:VERSION_ID="9" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:PLATFORM_ID="platform:el9" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:PRETTY_NAME="CentOS Stream 9" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:ANSI_COLOR="0;31" 2024-05-20T22:03:27.644 INFO:teuthology.orchestra.run.smithi078.stdout:LOGO="fedora-logo-icon" 2024-05-20T22:03:27.645 INFO:teuthology.orchestra.run.smithi078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-05-20T22:03:27.645 INFO:teuthology.orchestra.run.smithi078.stdout:HOME_URL="https://centos.org/" 2024-05-20T22:03:27.645 INFO:teuthology.orchestra.run.smithi078.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-05-20T22:03:27.645 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-05-20T22:03:27.645 INFO:teuthology.orchestra.run.smithi078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-05-20T22:03:27.645 INFO:teuthology.lock.ops:Updating smithi078.front.sepia.ceph.com on lock server 2024-05-20T22:03:27.662 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-05-20T22:03:27.671 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-05-20T22:03:27.731 INFO:teuthology.task.internal:Checking for old test directory... 2024-05-20T22:03:27.732 DEBUG:teuthology.orchestra.run.smithi006:> test '!' -e /home/ubuntu/cephtest 2024-05-20T22:03:27.734 DEBUG:teuthology.orchestra.run.smithi078:> test '!' -e /home/ubuntu/cephtest 2024-05-20T22:03:27.750 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-05-20T22:03:27.760 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-05-20T22:03:27.760 DEBUG:teuthology.orchestra.run.smithi006:> test -z $(ls -A /var/lib/ceph) 2024-05-20T22:03:27.792 DEBUG:teuthology.orchestra.run.smithi078:> test -z $(ls -A /var/lib/ceph) 2024-05-20T22:03:27.818 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-05-20T22:03:27.871 INFO:teuthology.run_tasks:Running task kernel... 2024-05-20T22:03:27.886 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-05-20T22:03:27.886 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-05-20T22:03:27.887 DEBUG:teuthology.orchestra.run.smithi006:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-20T22:03:27.887 DEBUG:teuthology.orchestra.run.smithi078:> test -f /run/.containerenv -o -f /.dockerenv 2024-05-20T22:03:27.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:03:27.903 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-05-20T22:03:27.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:03:27.904 DEBUG:teuthology.orchestra.run.smithi006:> uname -r 2024-05-20T22:03:27.960 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-437.el9.x86_64 2024-05-20T22:03:27.960 INFO:teuthology.task.kernel:Running kernel on smithi078: 5.14.0-437.el9.x86_64 2024-05-20T22:03:27.960 DEBUG:teuthology.orchestra.run.smithi078:> sudo yum install -y kernel 2024-05-20T22:03:27.962 INFO:teuthology.orchestra.run.smithi006.stdout:5.14.0-437.el9.x86_64 2024-05-20T22:03:27.962 INFO:teuthology.task.kernel:Running kernel on smithi006: 5.14.0-437.el9.x86_64 2024-05-20T22:03:27.962 DEBUG:teuthology.orchestra.run.smithi006:> sudo yum install -y kernel 2024-05-20T22:03:32.259 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - BaseOS 3.0 MB/s | 8.1 MB 00:02 2024-05-20T22:03:34.412 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - BaseOS 1.7 MB/s | 8.1 MB 00:04 2024-05-20T22:03:47.209 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - AppStream 1.4 MB/s | 19 MB 00:13 2024-05-20T22:03:53.144 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - AppStream 1.1 MB/s | 19 MB 00:17 2024-05-20T22:03:53.176 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - CRB 2.8 MB/s | 6.2 MB 00:02 2024-05-20T22:03:55.527 INFO:teuthology.orchestra.run.smithi006.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 16 kB 00:01 2024-05-20T22:03:57.786 INFO:teuthology.orchestra.run.smithi006.stdout:Extra Packages for Enterprise Linux 9.9 MB/s | 21 MB 00:02 2024-05-20T22:04:00.012 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - CRB 1.9 MB/s | 6.2 MB 00:03 2024-05-20T22:04:01.495 INFO:teuthology.orchestra.run.smithi078.stdout:CentOS Stream 9 - Extras packages 49 kB/s | 16 kB 00:00 2024-05-20T22:04:03.237 INFO:teuthology.orchestra.run.smithi006.stdout:lab-extras 26 kB/s | 1.7 kB 00:00 2024-05-20T22:04:03.449 INFO:teuthology.orchestra.run.smithi078.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-05-20T22:04:05.210 INFO:teuthology.orchestra.run.smithi006.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-20T22:04:05.210 INFO:teuthology.orchestra.run.smithi006.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-20T22:04:05.293 INFO:teuthology.orchestra.run.smithi006.stdout:Dependencies resolved. 2024-05-20T22:04:05.303 INFO:teuthology.orchestra.run.smithi006.stdout:================================================================================ 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout: Package Architecture Version Repository Size 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout:================================================================================ 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout:Installing: 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout: kernel x86_64 5.14.0-447.el9 baseos 576 k 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout:Installing dependencies: 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-core x86_64 5.14.0-447.el9 baseos 16 M 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules x86_64 5.14.0-447.el9 baseos 34 M 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules-core x86_64 5.14.0-447.el9 baseos 28 M 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout:Transaction Summary 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout:================================================================================ 2024-05-20T22:04:05.304 INFO:teuthology.orchestra.run.smithi006.stdout:Install 4 Packages 2024-05-20T22:04:05.305 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:04:05.305 INFO:teuthology.orchestra.run.smithi006.stdout:Total download size: 78 M 2024-05-20T22:04:05.305 INFO:teuthology.orchestra.run.smithi006.stdout:Installed size: 123 M 2024-05-20T22:04:05.305 INFO:teuthology.orchestra.run.smithi006.stdout:Downloading Packages: 2024-05-20T22:04:05.492 INFO:teuthology.orchestra.run.smithi006.stdout:[MIRROR] kernel-5.14.0-447.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-447.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T22:04:05.492 INFO:teuthology.orchestra.run.smithi006.stdout:[MIRROR] kernel-modules-5.14.0-447.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-447.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T22:04:05.492 INFO:teuthology.orchestra.run.smithi006.stdout:[MIRROR] kernel-core-5.14.0-447.el9.x86_64.rpm: Status code: 404 for http://mirror.siena.edu/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-447.el9.x86_64.rpm (IP: 199.223.246.113) 2024-05-20T22:04:06.180 INFO:teuthology.orchestra.run.smithi006.stdout:(1/4): kernel-5.14.0-447.el9.x86_64.rpm 798 kB/s | 576 kB 00:00 2024-05-20T22:04:08.522 INFO:teuthology.orchestra.run.smithi078.stdout:lab-extras 18 kB/s | 1.7 kB 00:00 2024-05-20T22:04:10.378 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-05-20T22:04:10.378 INFO:teuthology.orchestra.run.smithi078.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-05-20T22:04:10.468 INFO:teuthology.orchestra.run.smithi078.stdout:Dependencies resolved. 2024-05-20T22:04:10.479 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-05-20T22:04:10.479 INFO:teuthology.orchestra.run.smithi078.stdout: Package Architecture Version Repository Size 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout:Installing: 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout: kernel x86_64 5.14.0-447.el9 baseos 576 k 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout:Installing dependencies: 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core x86_64 5.14.0-447.el9 baseos 16 M 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules x86_64 5.14.0-447.el9 baseos 34 M 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core x86_64 5.14.0-447.el9 baseos 28 M 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction Summary 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout:================================================================================ 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout:Install 4 Packages 2024-05-20T22:04:10.480 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:04:10.481 INFO:teuthology.orchestra.run.smithi078.stdout:Total download size: 78 M 2024-05-20T22:04:10.481 INFO:teuthology.orchestra.run.smithi078.stdout:Installed size: 123 M 2024-05-20T22:04:10.481 INFO:teuthology.orchestra.run.smithi078.stdout:Downloading Packages: 2024-05-20T22:04:11.249 INFO:teuthology.orchestra.run.smithi078.stdout:(1/4): kernel-5.14.0-447.el9.x86_64.rpm 808 kB/s | 576 kB 00:00 2024-05-20T22:04:15.046 INFO:teuthology.orchestra.run.smithi006.stdout:(2/4): kernel-core-5.14.0-447.el9.x86_64.rpm 1.6 MB/s | 16 MB 00:09 2024-05-20T22:04:16.163 INFO:teuthology.orchestra.run.smithi006.stdout:(3/4): kernel-modules-5.14.0-447.el9.x86_64.rpm 3.2 MB/s | 34 MB 00:10 2024-05-20T22:04:21.239 INFO:teuthology.orchestra.run.smithi078.stdout:(2/4): kernel-core-5.14.0-447.el9.x86_64.rpm 1.4 MB/s | 16 MB 00:10 2024-05-20T22:04:38.420 INFO:teuthology.orchestra.run.smithi006.stdout:(4/4): kernel-modules-core-5.14.0-447.el9.x86_6 891 kB/s | 28 MB 00:32 2024-05-20T22:04:38.420 INFO:teuthology.orchestra.run.smithi006.stdout:-------------------------------------------------------------------------------- 2024-05-20T22:04:38.420 INFO:teuthology.orchestra.run.smithi006.stdout:Total 2.4 MB/s | 78 MB 00:33 2024-05-20T22:04:39.131 INFO:teuthology.orchestra.run.smithi006.stdout:Running transaction check 2024-05-20T22:04:39.561 INFO:teuthology.orchestra.run.smithi006.stdout:Transaction check succeeded. 2024-05-20T22:04:39.561 INFO:teuthology.orchestra.run.smithi006.stdout:Running transaction test 2024-05-20T22:04:40.533 INFO:teuthology.orchestra.run.smithi006.stdout:Transaction test succeeded. 2024-05-20T22:04:40.533 INFO:teuthology.orchestra.run.smithi006.stdout:Running transaction 2024-05-20T22:04:40.988 INFO:teuthology.orchestra.run.smithi078.stdout:(3/4): kernel-modules-core-5.14.0-447.el9.x86_6 966 kB/s | 28 MB 00:29 2024-05-20T22:04:43.064 INFO:teuthology.orchestra.run.smithi006.stdout: Preparing : 1/1 2024-05-20T22:04:43.700 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-modules-core-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:04:43.787 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:04:46.594 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:04:46.733 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:04:50.437 INFO:teuthology.orchestra.run.smithi078.stdout:(4/4): kernel-modules-5.14.0-447.el9.x86_64.rpm 874 kB/s | 34 MB 00:39 2024-05-20T22:04:50.437 INFO:teuthology.orchestra.run.smithi078.stdout:-------------------------------------------------------------------------------- 2024-05-20T22:04:50.437 INFO:teuthology.orchestra.run.smithi078.stdout:Total 2.0 MB/s | 78 MB 00:39 2024-05-20T22:04:51.120 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction check 2024-05-20T22:04:51.466 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction check succeeded. 2024-05-20T22:04:51.466 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction test 2024-05-20T22:04:51.812 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:04:51.853 INFO:teuthology.orchestra.run.smithi006.stdout: Installing : kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:04:52.388 INFO:teuthology.orchestra.run.smithi078.stdout:Transaction test succeeded. 2024-05-20T22:04:52.389 INFO:teuthology.orchestra.run.smithi078.stdout:Running transaction 2024-05-20T22:04:54.799 INFO:teuthology.orchestra.run.smithi078.stdout: Preparing : 1/1 2024-05-20T22:04:55.465 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-core-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:04:55.538 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:04:56.954 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:04:58.206 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:04:58.345 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:05:03.059 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:05:03.106 INFO:teuthology.orchestra.run.smithi078.stdout: Installing : kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:07.866 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:27.890 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:27.931 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:28.240 INFO:teuthology.orchestra.run.smithi006.stdout: Running scriptlet: kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:28.271 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:05:28.271 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:05:28.271 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:05:29.080 INFO:teuthology.orchestra.run.smithi006.stdout: Verifying : kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout:Installed: 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-5.14.0-447.el9.x86_64 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-core-5.14.0-447.el9.x86_64 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules-5.14.0-447.el9.x86_64 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout: kernel-modules-core-5.14.0-447.el9.x86_64 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:05:29.081 INFO:teuthology.orchestra.run.smithi006.stdout:Complete! 2024-05-20T22:05:29.453 DEBUG:teuthology.orchestra.run.smithi006:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:05:29.546 INFO:teuthology.orchestra.run.smithi006.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:05:29.546 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:05:29.547 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-447.el9.x86_64 2024-05-20T22:05:29.547 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-20T22:05:29.547 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-20T22:05:29.547 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-05-20T22:05:29.547 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi006.front.sepia.ceph.com, path=None, version=distro) 2024-05-20T22:05:29.547 DEBUG:teuthology.orchestra.run.smithi006:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:05:29.600 INFO:teuthology.orchestra.run.smithi006.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:05:29.600 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:05:29.600 DEBUG:teuthology.orchestra.run.smithi006:> sudo rpm -qi grub2-tools 2024-05-20T22:05:29.637 INFO:teuthology.orchestra.run.smithi006.stdout:Name : grub2-tools 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:Epoch : 1 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:Version : 2.06 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:Release : 68.el9 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:Architecture: x86_64 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:Group : Unspecified 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:Size : 8264254 2024-05-20T22:05:29.638 INFO:teuthology.orchestra.run.smithi006.stdout:License : GPLv3+ 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Packager : builder@centos.org 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Vendor : CentOS 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Summary : Support tools for GRUB. 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:Description : 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-20T22:05:29.639 INFO:teuthology.orchestra.run.smithi006.stdout:hardware devices. 2024-05-20T22:05:29.640 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:05:29.640 INFO:teuthology.orchestra.run.smithi006.stdout:This subpackage provides tools for support of all platforms. 2024-05-20T22:05:29.640 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-20T22:05:29.641 INFO:teuthology.task.kernel:Updating grub on smithi006 to boot 5.14.0-447.el9.x86_64 2024-05-20T22:05:29.641 DEBUG:teuthology.orchestra.run.smithi006:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-20T22:05:30.352 INFO:teuthology.orchestra.run.smithi006.stderr:Generating grub configuration file ... 2024-05-20T22:05:31.791 INFO:teuthology.orchestra.run.smithi006.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-20T22:05:31.810 INFO:teuthology.orchestra.run.smithi006.stderr:done 2024-05-20T22:05:31.812 DEBUG:teuthology.orchestra.run.smithi006:> mktemp 2024-05-20T22:05:31.868 INFO:teuthology.orchestra.run.smithi006.stdout:/tmp/tmp.sl4RYPNNIL 2024-05-20T22:05:31.868 DEBUG:teuthology.orchestra.run.smithi006:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.sl4RYPNNIL 2024-05-20T22:05:31.934 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 0666 /tmp/tmp.sl4RYPNNIL 2024-05-20T22:05:32.072 DEBUG:teuthology.orchestra.remote:smithi006:/tmp/tmp.sl4RYPNNIL is 6KB 2024-05-20T22:05:32.120 DEBUG:teuthology.orchestra.run.smithi006:> rm -fr /tmp/tmp.sl4RYPNNIL 2024-05-20T22:05:32.135 DEBUG:teuthology.orchestra.run.smithi006:> sudo /bin/ls /boot/loader/entries || true 2024-05-20T22:05:32.200 INFO:teuthology.orchestra.run.smithi006.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-20T22:05:32.200 INFO:teuthology.orchestra.run.smithi006.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-20T22:05:32.200 INFO:teuthology.orchestra.run.smithi006.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-20T22:05:32.200 INFO:teuthology.orchestra.run.smithi006.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64.conf 2024-05-20T22:05:32.201 DEBUG:teuthology.orchestra.run.smithi006:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64 2024-05-20T22:05:32.349 DEBUG:teuthology.orchestra.run.smithi006:> sudo shutdown -r now 2024-05-20T22:05:37.104 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:37.142 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-modules-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:37.509 INFO:teuthology.orchestra.run.smithi078.stdout: Running scriptlet: kernel-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:37.509 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-5.14.0-447.el9.x86_64 1/4 2024-05-20T22:05:37.509 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-core-5.14.0-447.el9.x86_64 2/4 2024-05-20T22:05:37.509 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-5.14.0-447.el9.x86_64 3/4 2024-05-20T22:05:38.294 INFO:teuthology.orchestra.run.smithi078.stdout: Verifying : kernel-modules-core-5.14.0-447.el9.x86_64 4/4 2024-05-20T22:05:38.294 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:05:38.295 INFO:teuthology.orchestra.run.smithi078.stdout:Installed: 2024-05-20T22:05:38.295 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-5.14.0-447.el9.x86_64 2024-05-20T22:05:38.295 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-core-5.14.0-447.el9.x86_64 2024-05-20T22:05:38.295 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-5.14.0-447.el9.x86_64 2024-05-20T22:05:38.295 INFO:teuthology.orchestra.run.smithi078.stdout: kernel-modules-core-5.14.0-447.el9.x86_64 2024-05-20T22:05:38.295 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:05:38.295 INFO:teuthology.orchestra.run.smithi078.stdout:Complete! 2024-05-20T22:05:38.615 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:05:38.687 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:05:38.687 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:05:38.687 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-447.el9.x86_64 2024-05-20T22:05:38.687 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-05-20T22:05:38.687 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-05-20T22:05:38.688 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-05-20T22:05:38.688 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi078.front.sepia.ceph.com, path=None, version=distro) 2024-05-20T22:05:38.688 DEBUG:teuthology.orchestra.run.smithi078:> rpm -q kernel | sort -rV | head -n 1 2024-05-20T22:05:38.736 INFO:teuthology.orchestra.run.smithi078.stdout:kernel-5.14.0-447.el9.x86_64 2024-05-20T22:05:38.737 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-447.el9.x86_64 2024-05-20T22:05:38.737 DEBUG:teuthology.orchestra.run.smithi078:> sudo rpm -qi grub2-tools 2024-05-20T22:05:38.767 INFO:teuthology.orchestra.run.smithi078.stdout:Name : grub2-tools 2024-05-20T22:05:38.767 INFO:teuthology.orchestra.run.smithi078.stdout:Epoch : 1 2024-05-20T22:05:38.767 INFO:teuthology.orchestra.run.smithi078.stdout:Version : 2.06 2024-05-20T22:05:38.767 INFO:teuthology.orchestra.run.smithi078.stdout:Release : 68.el9 2024-05-20T22:05:38.767 INFO:teuthology.orchestra.run.smithi078.stdout:Architecture: x86_64 2024-05-20T22:05:38.767 INFO:teuthology.orchestra.run.smithi078.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-05-20T22:05:38.767 INFO:teuthology.orchestra.run.smithi078.stdout:Group : Unspecified 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Size : 8264254 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:License : GPLv3+ 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Packager : builder@centos.org 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Vendor : CentOS 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:URL : http://www.gnu.org/software/grub/ 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Summary : Support tools for GRUB. 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:Description : 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:05:38.768 INFO:teuthology.orchestra.run.smithi078.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-05-20T22:05:38.769 INFO:teuthology.orchestra.run.smithi078.stdout:customizable bootloader with modular architecture. It supports a rich 2024-05-20T22:05:38.769 INFO:teuthology.orchestra.run.smithi078.stdout:variety of kernel formats, file systems, computer architectures and 2024-05-20T22:05:38.769 INFO:teuthology.orchestra.run.smithi078.stdout:hardware devices. 2024-05-20T22:05:38.769 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:05:38.769 INFO:teuthology.orchestra.run.smithi078.stdout:This subpackage provides tools for support of all platforms. 2024-05-20T22:05:38.770 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-05-20T22:05:38.770 INFO:teuthology.task.kernel:Updating grub on smithi078 to boot 5.14.0-447.el9.x86_64 2024-05-20T22:05:38.770 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-05-20T22:05:39.336 INFO:teuthology.orchestra.run.smithi078.stderr:Generating grub configuration file ... 2024-05-20T22:05:40.708 INFO:teuthology.orchestra.run.smithi078.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-05-20T22:05:40.745 INFO:teuthology.orchestra.run.smithi078.stderr:done 2024-05-20T22:05:40.747 DEBUG:teuthology.orchestra.run.smithi078:> mktemp 2024-05-20T22:05:40.765 INFO:teuthology.orchestra.run.smithi078.stdout:/tmp/tmp.AHHeSGBSy1 2024-05-20T22:05:40.766 DEBUG:teuthology.orchestra.run.smithi078:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.AHHeSGBSy1 2024-05-20T22:05:40.833 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 0666 /tmp/tmp.AHHeSGBSy1 2024-05-20T22:05:40.967 DEBUG:teuthology.orchestra.remote:smithi078:/tmp/tmp.AHHeSGBSy1 is 6KB 2024-05-20T22:05:41.016 DEBUG:teuthology.orchestra.run.smithi078:> rm -fr /tmp/tmp.AHHeSGBSy1 2024-05-20T22:05:41.030 DEBUG:teuthology.orchestra.run.smithi078:> sudo /bin/ls /boot/loader/entries || true 2024-05-20T22:05:41.093 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-05-20T22:05:41.093 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-05-20T22:05:41.093 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-05-20T22:05:41.093 INFO:teuthology.orchestra.run.smithi078.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64.conf 2024-05-20T22:05:41.095 DEBUG:teuthology.orchestra.run.smithi078:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-447.el9.x86_64 2024-05-20T22:05:41.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo shutdown -r now 2024-05-20T22:06:02.370 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-05-20T22:06:02.371 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-05-20T22:06:02.372 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:06:11.254 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-05-20T22:06:11.255 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-20T22:06:11.255 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:06:20.706 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.6 2024-05-20T22:06:29.667 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-05-20T22:06:29.707 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-05-20T22:06:29.708 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:06:38.671 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-20T22:06:38.672 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:06:41.730 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.78 2024-05-20T22:06:53.742 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-20T22:06:53.743 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:07:25.529 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-05-20T22:07:25.960 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-20T22:07:25.960 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-447.el9.x86_64"... 2024-05-20T22:07:25.961 DEBUG:teuthology.orchestra.run.smithi078:> uname -r 2024-05-20T22:07:25.975 INFO:teuthology.orchestra.run.smithi078.stdout:5.14.0-447.el9.x86_64 2024-05-20T22:07:25.976 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-447.el9.x86_64 vs 5.14.0-447.el9.x86_64 2024-05-20T22:07:25.976 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-20T22:07:25.976 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-20T22:07:26.977 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-05-20T22:07:26.977 DEBUG:teuthology.orchestra.run.smithi078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-20T22:07:27.066 INFO:teuthology.orchestra.run.smithi078.stdout:ttyS1 2024-05-20T22:07:27.095 DEBUG:teuthology.parallel:result is None 2024-05-20T22:07:29.711 DEBUG:teuthology.orchestra.remote:timed out 2024-05-20T22:07:41.712 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-05-20T22:07:41.713 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:07:42.087 DEBUG:teuthology.orchestra.run.smithi006:> true 2024-05-20T22:07:42.446 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-05-20T22:07:42.447 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-447.el9.x86_64"... 2024-05-20T22:07:42.447 DEBUG:teuthology.orchestra.run.smithi006:> uname -r 2024-05-20T22:07:42.463 INFO:teuthology.orchestra.run.smithi006.stdout:5.14.0-447.el9.x86_64 2024-05-20T22:07:42.463 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-447.el9.x86_64 vs 5.14.0-447.el9.x86_64 2024-05-20T22:07:42.464 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-05-20T22:07:42.464 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-05-20T22:07:43.465 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-05-20T22:07:43.465 DEBUG:teuthology.orchestra.run.smithi006:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-05-20T22:07:43.536 INFO:teuthology.orchestra.run.smithi006.stdout:ttyS1 2024-05-20T22:07:43.565 DEBUG:teuthology.parallel:result is None 2024-05-20T22:07:43.565 INFO:teuthology.run_tasks:Running task internal.base... 2024-05-20T22:07:43.575 INFO:teuthology.task.internal:Creating test directory... 2024-05-20T22:07:43.576 DEBUG:teuthology.orchestra.run.smithi006:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-20T22:07:43.578 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-05-20T22:07:43.597 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-05-20T22:07:43.608 INFO:teuthology.run_tasks:Running task internal.archive... 2024-05-20T22:07:43.621 INFO:teuthology.task.internal:Creating archive directory... 2024-05-20T22:07:43.621 DEBUG:teuthology.orchestra.run.smithi006:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-20T22:07:43.637 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-05-20T22:07:43.690 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-05-20T22:07:43.700 INFO:teuthology.task.internal:Enabling coredump saving... 2024-05-20T22:07:43.701 DEBUG:teuthology.orchestra.run.smithi006:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-20T22:07:43.711 DEBUG:teuthology.orchestra.run.smithi078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-05-20T22:07:43.750 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:07:43.751 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:07:43.764 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:07:43.765 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-05-20T22:07:43.767 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-05-20T22:07:43.778 INFO:teuthology.task.internal:Configuring sudo... 2024-05-20T22:07:43.778 DEBUG:teuthology.orchestra.run.smithi006:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-20T22:07:43.809 DEBUG:teuthology.orchestra.run.smithi078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-05-20T22:07:43.838 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-05-20T22:07:43.851 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-05-20T22:07:43.852 DEBUG:teuthology.orchestra.run.smithi006:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-20T22:07:43.878 DEBUG:teuthology.orchestra.run.smithi078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-05-20T22:07:43.896 DEBUG:teuthology.orchestra.run.smithi006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:07:43.977 DEBUG:teuthology.orchestra.run.smithi006:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:07:44.028 DEBUG:teuthology.orchestra.run.smithi006:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:07:44.113 DEBUG:teuthology.orchestra.run.smithi006:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:07:44.155 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:07:44.155 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-20T22:07:44.221 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:07:44.263 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-05-20T22:07:44.315 DEBUG:teuthology.orchestra.run.smithi078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:07:44.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-05-20T22:07:44.440 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-20T22:07:44.441 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-05-20T22:07:44.506 DEBUG:teuthology.orchestra.run.smithi006:> sudo service rsyslog restart 2024-05-20T22:07:44.509 DEBUG:teuthology.orchestra.run.smithi078:> sudo service rsyslog restart 2024-05-20T22:07:44.559 INFO:teuthology.orchestra.run.smithi006.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T22:07:44.595 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T22:07:45.104 INFO:teuthology.run_tasks:Running task internal.timer... 2024-05-20T22:07:45.113 INFO:teuthology.task.internal:Starting timer... 2024-05-20T22:07:45.114 INFO:teuthology.run_tasks:Running task pcp... 2024-05-20T22:07:45.157 INFO:teuthology.run_tasks:Running task selinux... 2024-05-20T22:07:45.169 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-05-20T22:07:45.170 DEBUG:teuthology.orchestra.run.smithi006:> sudo service auditd rotate 2024-05-20T22:07:45.233 INFO:teuthology.orchestra.run.smithi006.stdout:Rotating logs: 2024-05-20T22:07:45.235 DEBUG:teuthology.orchestra.run.smithi078:> sudo service auditd rotate 2024-05-20T22:07:45.289 INFO:teuthology.orchestra.run.smithi078.stdout:Rotating logs: 2024-05-20T22:07:45.291 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-05-20T22:07:45.291 DEBUG:teuthology.orchestra.run.smithi006:> /usr/sbin/getenforce 2024-05-20T22:07:45.322 INFO:teuthology.orchestra.run.smithi006.stdout:Permissive 2024-05-20T22:07:45.322 DEBUG:teuthology.orchestra.run.smithi078:> /usr/sbin/getenforce 2024-05-20T22:07:45.354 INFO:teuthology.orchestra.run.smithi078.stdout:Permissive 2024-05-20T22:07:45.354 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi006.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi078.front.sepia.ceph.com': 'permissive'} 2024-05-20T22:07:45.355 DEBUG:teuthology.orchestra.run.smithi006:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T22:07:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:07:45.392 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T22:07:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:07:45.420 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-05-20T22:07:45.420 DEBUG:teuthology.orchestra.run.smithi006:> sudo /usr/sbin/setenforce permissive 2024-05-20T22:07:45.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo /usr/sbin/setenforce permissive 2024-05-20T22:07:45.486 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-05-20T22:07:45.495 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-05-20T22:07:45.507 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-05-20T22:07:45.508 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi006.front.sepia.ceph.com,smithi078.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-05-20T22:12:09.812 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi006.front.sepia.ceph.com'), Remote(name='ubuntu@smithi078.front.sepia.ceph.com')] 2024-05-20T22:12:09.814 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-05-20T22:12:09.815 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi006.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:12:09.894 DEBUG:teuthology.orchestra.run.smithi006:> true 2024-05-20T22:12:09.980 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi006.front.sepia.ceph.com' 2024-05-20T22:12:09.980 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-20T22:12:09.981 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-05-20T22:12:10.056 DEBUG:teuthology.orchestra.run.smithi078:> true 2024-05-20T22:12:10.138 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi078.front.sepia.ceph.com' 2024-05-20T22:12:10.138 INFO:teuthology.run_tasks:Running task clock... 2024-05-20T22:12:10.149 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-05-20T22:12:10.149 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-20T22:12:10.150 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T22:12:10.152 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-05-20T22:12:10.152 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T22:12:10.184 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-20T22:12:10.200 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-20T22:12:10.209 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-05-20T22:12:10.223 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-05-20T22:12:10.227 INFO:teuthology.orchestra.run.smithi006.stderr:sudo: ntpd: command not found 2024-05-20T22:12:10.239 INFO:teuthology.orchestra.run.smithi006.stdout:506 Cannot talk to daemon 2024-05-20T22:12:10.247 INFO:teuthology.orchestra.run.smithi078.stderr:sudo: ntpd: command not found 2024-05-20T22:12:10.254 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-20T22:12:10.258 INFO:teuthology.orchestra.run.smithi078.stdout:506 Cannot talk to daemon 2024-05-20T22:12:10.269 INFO:teuthology.orchestra.run.smithi006.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-20T22:12:10.273 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-05-20T22:12:10.286 INFO:teuthology.orchestra.run.smithi078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-05-20T22:12:10.326 INFO:teuthology.orchestra.run.smithi006.stderr:bash: line 1: ntpq: command not found 2024-05-20T22:12:10.330 INFO:teuthology.orchestra.run.smithi006.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T22:12:10.330 INFO:teuthology.orchestra.run.smithi006.stdout:=============================================================================== 2024-05-20T22:12:10.330 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.330 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.330 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.330 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.347 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-05-20T22:12:10.351 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T22:12:10.351 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-05-20T22:12:10.351 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.351 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.351 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.351 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:12:10.351 INFO:teuthology.run_tasks:Running task cephadm... 2024-05-20T22:12:10.448 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '8a373da3c102b503bf0a94126977cebf498afc8b'} 2024-05-20T22:12:10.448 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-05-20T22:12:10.449 INFO:tasks.cephadm:Cluster fsid is 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:12:10.449 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-05-20T22:12:10.449 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-05-20T22:12:10.449 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi006': '172.21.15.6', 'mon.smithi078': '172.21.15.78'} 2024-05-20T22:12:10.449 INFO:tasks.cephadm:Normalizing hostnames... 2024-05-20T22:12:10.449 DEBUG:teuthology.orchestra.run.smithi006:> sudo hostname $(hostname -s) 2024-05-20T22:12:10.476 DEBUG:teuthology.orchestra.run.smithi078:> sudo hostname $(hostname -s) 2024-05-20T22:12:10.501 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-05-20T22:12:10.501 DEBUG:teuthology.orchestra.run.smithi006:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-20T22:12:10.820 INFO:teuthology.orchestra.run.smithi006.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 20 22:12 /home/ubuntu/cephtest/cephadm 2024-05-20T22:12:10.821 DEBUG:teuthology.orchestra.run.smithi078:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-05-20T22:12:10.990 INFO:teuthology.orchestra.run.smithi078.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 May 20 22:12 /home/ubuntu/cephtest/cephadm 2024-05-20T22:12:10.990 DEBUG:teuthology.orchestra.run.smithi006:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-20T22:12:11.007 DEBUG:teuthology.orchestra.run.smithi078:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-05-20T22:12:11.031 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-05-20T22:12:11.032 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-20T22:12:11.050 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-05-20T22:12:11.899 INFO:teuthology.orchestra.run.smithi078.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:12:11.924 INFO:teuthology.orchestra.run.smithi006.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:12:41.483 INFO:teuthology.orchestra.run.smithi078.stdout:{ 2024-05-20T22:12:41.483 INFO:teuthology.orchestra.run.smithi078.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-20T22:12:41.484 INFO:teuthology.orchestra.run.smithi078.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-20T22:12:41.484 INFO:teuthology.orchestra.run.smithi078.stdout: "repo_digests": [ 2024-05-20T22:12:41.484 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-20T22:12:41.484 INFO:teuthology.orchestra.run.smithi078.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-20T22:12:41.484 INFO:teuthology.orchestra.run.smithi078.stdout: ] 2024-05-20T22:12:41.484 INFO:teuthology.orchestra.run.smithi078.stdout:} 2024-05-20T22:12:42.121 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:12:42.121 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-05-20T22:12:42.121 INFO:teuthology.orchestra.run.smithi006.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-05-20T22:12:42.121 INFO:teuthology.orchestra.run.smithi006.stdout: "repo_digests": [ 2024-05-20T22:12:42.121 INFO:teuthology.orchestra.run.smithi006.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-05-20T22:12:42.121 INFO:teuthology.orchestra.run.smithi006.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-05-20T22:12:42.121 INFO:teuthology.orchestra.run.smithi006.stdout: ] 2024-05-20T22:12:42.122 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:12:42.140 DEBUG:teuthology.orchestra.run.smithi006:> sudo mkdir -p /etc/ceph 2024-05-20T22:12:42.181 DEBUG:teuthology.orchestra.run.smithi078:> sudo mkdir -p /etc/ceph 2024-05-20T22:12:42.209 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 777 /etc/ceph 2024-05-20T22:12:42.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo chmod 777 /etc/ceph 2024-05-20T22:12:42.275 INFO:tasks.cephadm:Writing seed config... 2024-05-20T22:12:42.278 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-05-20T22:12:42.278 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-05-20T22:12:42.278 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-05-20T22:12:42.279 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-05-20T22:12:42.279 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-05-20T22:12:42.279 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-05-20T22:12:42.279 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-05-20T22:12:42.279 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-05-20T22:12:42.279 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:12:42.279 DEBUG:teuthology.orchestra.run.smithi006:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-05-20T22:12:42.301 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 00e42188-16f6-11ef-bc9a-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-05-20T22:12:42.301 DEBUG:teuthology.orchestra.run.smithi006:mon.smithi006> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service 2024-05-20T22:12:42.344 INFO:tasks.cephadm:Bootstrapping... 2024-05-20T22:12:42.344 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.6 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:12:42.559 INFO:teuthology.orchestra.run.smithi006.stderr:-------------------------------------------------------------------------------- 2024-05-20T22:12:42.559 INFO:teuthology.orchestra.run.smithi006.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '00e42188-16f6-11ef-bc9a-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.6', '--skip-admin-label'] 2024-05-20T22:12:42.584 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: 5.0.2 2024-05-20T22:12:42.586 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying podman|docker is present... 2024-05-20T22:12:42.610 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: 5.0.2 2024-05-20T22:12:42.612 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying lvm2 is present... 2024-05-20T22:12:42.612 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying time synchronization is in place... 2024-05-20T22:12:42.619 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-20T22:12:42.625 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-05-20T22:12:42.633 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: enabled 2024-05-20T22:12:42.639 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: active 2024-05-20T22:12:42.640 INFO:teuthology.orchestra.run.smithi006.stderr:Unit chronyd.service is enabled and running 2024-05-20T22:12:42.640 INFO:teuthology.orchestra.run.smithi006.stderr:Repeating the final host check... 2024-05-20T22:12:42.663 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: 5.0.2 2024-05-20T22:12:42.664 INFO:teuthology.orchestra.run.smithi006.stderr:podman (/bin/podman) version 5.0.2 is present 2024-05-20T22:12:42.665 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl is present 2024-05-20T22:12:42.665 INFO:teuthology.orchestra.run.smithi006.stderr:lvcreate is present 2024-05-20T22:12:42.671 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-05-20T22:12:42.678 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-05-20T22:12:42.685 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: enabled 2024-05-20T22:12:42.692 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: active 2024-05-20T22:12:42.693 INFO:teuthology.orchestra.run.smithi006.stderr:Unit chronyd.service is enabled and running 2024-05-20T22:12:42.694 INFO:teuthology.orchestra.run.smithi006.stderr:Host looks OK 2024-05-20T22:12:42.694 INFO:teuthology.orchestra.run.smithi006.stderr:Cluster fsid: 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:12:42.694 INFO:teuthology.orchestra.run.smithi006.stderr:Acquiring lock 140051880148848 on /run/cephadm/00e42188-16f6-11ef-bc9a-c7b262605968.lock 2024-05-20T22:12:42.694 INFO:teuthology.orchestra.run.smithi006.stderr:Lock 140051880148848 acquired on /run/cephadm/00e42188-16f6-11ef-bc9a-c7b262605968.lock 2024-05-20T22:12:42.694 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying IP 172.21.15.6 port 3300 ... 2024-05-20T22:12:42.695 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying IP 172.21.15.6 port 6789 ... 2024-05-20T22:12:42.695 INFO:teuthology.orchestra.run.smithi006.stderr:Base mon IP is 172.21.15.6, final addrv is [v2:172.21.15.6:3300,v1:172.21.15.6:6789] 2024-05-20T22:12:42.697 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.6 metric 100 2024-05-20T22:12:42.697 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.6 metric 100 2024-05-20T22:12:42.700 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-05-20T22:12:42.700 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-05-20T22:12:42.700 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-05-20T22:12:42.703 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-05-20T22:12:42.703 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-05-20T22:12:42.703 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-20T22:12:42.703 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-05-20T22:12:42.703 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:15e4/64 scope link noprefixroute 2024-05-20T22:12:42.703 INFO:teuthology.orchestra.run.smithi006.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-05-20T22:12:42.704 INFO:teuthology.orchestra.run.smithi006.stderr:Mon IP `172.21.15.6` is in CIDR network `172.21.0.0/20` 2024-05-20T22:12:42.704 INFO:teuthology.orchestra.run.smithi006.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-05-20T22:12:42.704 INFO:teuthology.orchestra.run.smithi006.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:12:42.809 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-05-20T22:12:43.175 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Getting image source signatures 2024-05-20T22:12:43.175 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-05-20T22:12:43.175 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-05-20T22:12:43.175 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-05-20T22:12:43.175 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-05-20T22:12:43.175 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-05-20T22:12:43.175 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-20T22:12:43.316 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: Writing manifest to image destination 2024-05-20T22:12:43.435 INFO:teuthology.orchestra.run.smithi006.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-05-20T22:12:43.830 INFO:teuthology.orchestra.run.smithi006.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-20T22:12:44.394 INFO:teuthology.orchestra.run.smithi006.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-05-20T22:12:44.395 INFO:teuthology.orchestra.run.smithi006.stderr:Extracting ceph user uid/gid from container image... 2024-05-20T22:12:44.777 INFO:teuthology.orchestra.run.smithi006.stderr:stat: 167 167 2024-05-20T22:12:45.358 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial keys... 2024-05-20T22:12:45.773 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQDdyktmRQ4xKRAAsg3MUJfrFRJFbW+p/seh/g== 2024-05-20T22:12:46.854 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQDeyktmend7LhAAoP0VXud/Vh39W1K3U1B6iQ== 2024-05-20T22:12:47.735 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph-authtool: AQDfyktmazTdJhAAXekaftwCQDa2J6tShhXH5w== 2024-05-20T22:12:48.342 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial monmap... 2024-05-20T22:12:48.734 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-20T22:12:48.734 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-05-20T22:12:48.734 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:12:48.734 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-20T22:12:49.207 INFO:teuthology.orchestra.run.smithi006.stderr:monmaptool for smithi006 [v2:172.21.15.6:3300,v1:172.21.15.6:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-05-20T22:12:49.207 INFO:teuthology.orchestra.run.smithi006.stderr:setting min_mon_release = octopus 2024-05-20T22:12:49.207 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: set fsid to 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:12:49.207 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-05-20T22:12:49.207 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:12:49.208 INFO:teuthology.orchestra.run.smithi006.stderr:Creating mon... 2024-05-20T22:12:50.313 INFO:teuthology.orchestra.run.smithi006.stderr:create mon.smithi006 on 2024-05-20T22:12:50.529 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-05-20T22:12:50.682 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-00e42188-16f6-11ef-bc9a-c7b262605968.target → /etc/systemd/system/ceph-00e42188-16f6-11ef-bc9a-c7b262605968.target. 2024-05-20T22:12:50.683 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-00e42188-16f6-11ef-bc9a-c7b262605968.target → /etc/systemd/system/ceph-00e42188-16f6-11ef-bc9a-c7b262605968.target. 2024-05-20T22:12:51.112 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to reset failed state of unit ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service: Unit ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service not loaded. 2024-05-20T22:12:51.120 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph-00e42188-16f6-11ef-bc9a-c7b262605968.target.wants/ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service → /etc/systemd/system/ceph-00e42188-16f6-11ef-bc9a-c7b262605968@.service. 2024-05-20T22:12:51.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:51 smithi006 podman[24434]: 2024-05-20 22:12:51.626429741 +0000 UTC m=+0.018405271 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-20T22:12:52.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:52 smithi006 podman[24434]: 2024-05-20 22:12:52.498633033 +0000 UTC m=+0.890608557 container create d351cdf60b0fc51689e54a8d3258dfd6e16cda59c65e36e43329112892a2ba98 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, ceph=True, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-20T22:12:54.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:54 smithi006 podman[24434]: 2024-05-20 22:12:54.235938792 +0000 UTC m=+2.627914325 container init d351cdf60b0fc51689e54a8d3258dfd6e16cda59c65e36e43329112892a2ba98 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, version=8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, release=754) 2024-05-20T22:12:54.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:54 smithi006 podman[24434]: 2024-05-20 22:12:54.240401878 +0000 UTC m=+2.632377409 container start d351cdf60b0fc51689e54a8d3258dfd6e16cda59c65e36e43329112892a2ba98 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, ceph=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.openshift.expose-services=, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8) 2024-05-20T22:12:54.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:54 smithi006 ceph-mon[24452]: mkfs 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:12:54.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:54 smithi006 ceph-mon[24452]: mon.smithi006 is new leader, mons smithi006 in quorum (ranks 0) 2024-05-20T22:12:55.102 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-05-20T22:12:55.110 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-05-20T22:12:55.111 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-05-20T22:12:55.111 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to enable service . firewalld.service is not available 2024-05-20T22:12:55.112 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mon to start... 2024-05-20T22:12:55.112 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mon... 2024-05-20T22:12:55.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:55 smithi006 bash[24434]: d351cdf60b0fc51689e54a8d3258dfd6e16cda59c65e36e43329112892a2ba98 2024-05-20T22:12:55.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:55 smithi006 systemd[1]: Started Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:12:55.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:55 smithi006 ceph-mon[24452]: mon.smithi006 is new leader, mons smithi006 in quorum (ranks 0) 2024-05-20T22:12:55.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:55 smithi006 ceph-mon[24452]: monmap e1: 1 mons at {smithi006=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0]} 2024-05-20T22:12:55.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:55 smithi006 ceph-mon[24452]: fsmap 2024-05-20T22:12:55.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:55 smithi006 ceph-mon[24452]: osdmap e1: 0 total, 0 up, 0 in 2024-05-20T22:12:55.389 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:55 smithi006 ceph-mon[24452]: mgrmap e1: no daemons active 2024-05-20T22:12:58.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:12:58 smithi006 ceph-mon[24452]: from='client.? 172.21.15.6:0/3769298897' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-05-20T22:12:59.157 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: cluster: 2024-05-20T22:12:59.157 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: id: 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:12:59.157 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-05-20T22:12:59.157 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:12:59.157 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: services: 2024-05-20T22:12:59.158 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi006 (age 4s) 2024-05-20T22:12:59.158 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr: no daemons active 2024-05-20T22:12:59.158 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-05-20T22:12:59.158 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:12:59.158 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: data: 2024-05-20T22:12:59.159 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-05-20T22:12:59.159 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-05-20T22:12:59.159 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-05-20T22:12:59.159 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: pgs: 2024-05-20T22:12:59.159 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:01.421 INFO:teuthology.orchestra.run.smithi006.stderr:mon is available 2024-05-20T22:13:01.421 INFO:teuthology.orchestra.run.smithi006.stderr:Assimilating anything we can from ceph.conf... 2024-05-20T22:13:02.679 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:02.679 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [global] 2024-05-20T22:13:02.679 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: fsid = 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.6:3300,v1:172.21.15.6:6789] 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [mgr] 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [osd] 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-20T22:13:02.680 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-20T22:13:02.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:02 smithi006 ceph-mon[24452]: from='client.? 172.21.15.6:0/2466985244' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-20T22:13:02.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:02 smithi006 ceph-mon[24452]: from='client.? 172.21.15.6:0/2466985244' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-05-20T22:13:03.277 INFO:teuthology.orchestra.run.smithi006.stderr:Generating new minimal ceph.conf... 2024-05-20T22:13:04.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:03 smithi006 ceph-mon[24452]: from='client.? 172.21.15.6:0/271831085' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:13:04.300 INFO:teuthology.orchestra.run.smithi006.stderr:Restarting the monitor... 2024-05-20T22:13:04.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 systemd[1]: Stopping Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:13:04.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 bash[24674]: Error: no container with name or ID "ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon.smithi006" found: no such container 2024-05-20T22:13:04.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[24448]: 2024-05-20T22:13:04.457+0000 7ffbba6c0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi006 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:13:04.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[24448]: 2024-05-20T22:13:04.457+0000 7ffbba6c0700 -1 mon.smithi006@0(leader) e1 *** Got Signal Terminated *** 2024-05-20T22:13:04.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 podman[24685]: 2024-05-20 22:13:04.489237245 +0000 UTC m=+0.125279766 container died d351cdf60b0fc51689e54a8d3258dfd6e16cda59c65e36e43329112892a2ba98 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, distribution-scope=public, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-20T22:13:05.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 podman[24685]: 2024-05-20 22:13:04.824221626 +0000 UTC m=+0.460264148 container cleanup d351cdf60b0fc51689e54a8d3258dfd6e16cda59c65e36e43329112892a2ba98 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, ceph=True, architecture=x86_64, release=754, io.openshift.expose-services=, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:13:05.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 bash[24685]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006 2024-05-20T22:13:05.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 podman[24698]: 2024-05-20 22:13:04.975118793 +0000 UTC m=+0.489608679 container remove d351cdf60b0fc51689e54a8d3258dfd6e16cda59c65e36e43329112892a2ba98 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, architecture=x86_64, GIT_CLEAN=True, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-20T22:13:05.000 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:04 smithi006 bash[24710]: Error: no container with name or ID "ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon.smithi006" found: no such container 2024-05-20T22:13:05.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service: Deactivated successfully. 2024-05-20T22:13:05.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 systemd[1]: Stopped Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:13:05.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 systemd[1]: Starting Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:13:05.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 podman[24772]: 2024-05-20 22:13:05.200960748 +0000 UTC m=+0.089766608 container create 8dde577b64917d8e8766480b90a3fb1b9d06f0da1d83a9168fb1c1fe399551bb (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, architecture=x86_64, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.buildah.version=1.19.8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD) 2024-05-20T22:13:05.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 podman[24772]: 2024-05-20 22:13:05.129505911 +0000 UTC m=+0.018311798 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-05-20T22:13:05.523 INFO:teuthology.orchestra.run.smithi006.stderr:Setting mon public_network to 172.21.0.0/20 2024-05-20T22:13:05.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 podman[24772]: 2024-05-20 22:13:05.453168474 +0000 UTC m=+0.341974349 container init 8dde577b64917d8e8766480b90a3fb1b9d06f0da1d83a9168fb1c1fe399551bb (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.buildah.version=1.19.8, version=8, name=centos-stream, io.openshift.tags=base centos centos-stream) 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 podman[24772]: 2024-05-20 22:13:05.456376705 +0000 UTC m=+0.345182572 container start 8dde577b64917d8e8766480b90a3fb1b9d06f0da1d83a9168fb1c1fe399551bb (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-type=git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_CLEAN=True, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux ) 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: pidfile_write: ignore empty --pid-file 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: load: jerasure load: lrc 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: RocksDB version: 6.15.5 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Compile date Apr 18 2022 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: DB SUMMARY 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: DB Session ID: 8MTWIVLD05VR4HXRGZW4 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:13:05.647 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi006/store.db dir, Total Num: 1, files: 000008.sst 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi006/store.db: 000010.log size: 83978 ; 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.env: 0x555f0a0c6860 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.fs: Posix File System 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.info_log: 0x555f0b7cdfa0 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.statistics: (nil) 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.use_fsync: 0 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:13:05.648 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.db_log_dir: 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi006/store.db 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:13:05.649 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:13:05.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:13:05.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:13:05.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:13:05.650 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:13:05.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:13:05.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.write_buffer_manager: 0x555f0b8be330 2024-05-20T22:13:05.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:13:05.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-20T22:13:05.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:13:05.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:13:05.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.unordered_write: 0 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.row_cache: None 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.wal_filter: None 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:13:05.652 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.preserve_deletes: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:13:05.653 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_open_files: -1 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:13:05.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Compression algorithms supported: 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kZSTD supported: 0 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000009 2024-05-20T22:13:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.merge_operator: 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_filter: None 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555f0b799d00) 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_type: 0 2024-05-20T22:13:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_index_type: 0 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_shortening: 1 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: hash_index_allow_collision: 1 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: checksum: 1 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: no_block_cache: 0 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache: 0x555f0b804170 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_options: 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: capacity : 536870912 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: num_shard_bits : 4 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: strict_capacity_limit : 0 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_compressed: (nil) 2024-05-20T22:13:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: persistent_cache: (nil) 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size: 4096 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size_deviation: 10 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_restart_interval: 16 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_block_restart_interval: 1 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: metadata_block_size: 4096 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: partition_filters: 0 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: use_delta_encoding: 1 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: whole_key_filtering: 1 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: verify_compression: 0 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: format_version: 4 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_index_compression: 1 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_align: 0 2024-05-20T22:13:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression: NoCompression 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.num_levels: 7 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:13:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:13:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.arena_block_size: 4194304 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:13:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:13:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.ttl: 2592000 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:13:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243185482513, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243185483297, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80063, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78308, "index_size": 243, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9946, "raw_average_key_size": 46, "raw_value_size": 72545, "raw_average_value_size": 340, "num_data_blocks": 11, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1716243185, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a2236df6-5be1-446b-928d-5a7ef87f2516", "db_session_id": "8MTWIVLD05VR4HXRGZW4"}} 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243185483651, "job": 1, "event": "recovery_finished"} 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi006/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:13:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x555f0b7eaa80 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: DB pointer 0x555f0b7fa000 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** DB Stats ** 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:13:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L0 2/0 79.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.8 0.00 0.00 1 0.001 0 0 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Sum 2/0 79.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.8 0.00 0.00 1 0.001 0 0 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.8 0.00 0.00 1 0.001 0 0 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.8 0.00 0.00 1 0.001 0 0 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:13:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative compaction: 0.00 GB write, 24.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval compaction: 0.00 GB write, 24.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L0 2/0 79.98 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.8 0.00 0.00 1 0.001 0 0 2024-05-20T22:13:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Sum 2/0 79.98 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.8 0.00 0.00 1 0.001 0 0 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.8 0.00 0.00 1 0.001 0 0 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative compaction: 0.00 GB write, 24.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:13:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: starting mon.smithi006 rank 0 at public addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] at bind addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi006 fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???) e1 preinit fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???).mds e1 new map 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???).mds e1 print_map 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: e1 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: legacy client fscid: -1 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:13:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: No filesystems configured 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 bash[24772]: 8dde577b64917d8e8766480b90a3fb1b9d06f0da1d83a9168fb1c1fe399551bb 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 systemd[1]: Started Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mon.smithi006 is new leader, mons smithi006 in quorum (ranks 0) 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: monmap e1: 1 mons at {smithi006=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0]} 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: fsmap 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: osdmap e1: 0 total, 0 up, 0 in 2024-05-20T22:13:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:05 smithi006 ceph-mon[24790]: mgrmap e1: no daemons active 2024-05-20T22:13:07.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:07 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1798470922' entity='client.admin' 2024-05-20T22:13:07.713 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote config to /etc/ceph/ceph.conf 2024-05-20T22:13:07.713 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:13:07.713 INFO:teuthology.orchestra.run.smithi006.stderr:Creating mgr... 2024-05-20T22:13:07.713 INFO:teuthology.orchestra.run.smithi006.stderr:Verifying port 9283 ... 2024-05-20T22:13:07.903 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Failed to reset failed state of unit ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mgr.smithi006.ssyogp.service: Unit ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mgr.smithi006.ssyogp.service not loaded. 2024-05-20T22:13:07.910 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: Created symlink /etc/systemd/system/ceph-00e42188-16f6-11ef-bc9a-c7b262605968.target.wants/ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mgr.smithi006.ssyogp.service → /etc/systemd/system/ceph-00e42188-16f6-11ef-bc9a-c7b262605968@.service. 2024-05-20T22:13:08.568 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-05-20T22:13:08.576 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-05-20T22:13:08.577 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-05-20T22:13:08.577 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to enable service . firewalld.service is not available 2024-05-20T22:13:08.587 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-05-20T22:13:08.595 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-05-20T22:13:08.596 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-05-20T22:13:08.596 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-05-20T22:13:08.596 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr to start... 2024-05-20T22:13:08.596 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr... 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "00e42188-16f6-11ef-bc9a-c7b262605968", 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:13:09.097 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 3, 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:13:09.098 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:13:09.099 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:13:09.100 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:12:54.284834+0000", 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:13:09.101 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:09.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:09 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3358436256' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:13:09.696 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (1/15)... 2024-05-20T22:13:12.269 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:12.269 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "00e42188-16f6-11ef-bc9a-c7b262605968", 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-05-20T22:13:12.270 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:13:12.271 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:13:12.275 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:12.276 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:12:54.284834+0000", 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:13:12.277 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:12.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:12 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/779517237' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:13:12.861 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (2/15)... 2024-05-20T22:13:15.095 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: Activating manager daemon smithi006.ssyogp 2024-05-20T22:13:15.095 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: mgrmap e2: smithi006.ssyogp(active, starting, since 0.00268227s) 2024-05-20T22:13:15.095 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:13:15.095 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:13:15.095 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:13:15.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:13:15.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:13:15.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:13:15.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:13:15.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:13:15.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:15.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:15.097 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:14 smithi006 ceph-mon[24790]: from='mgr.14100 172.21.15.6:0/953858342' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:15.387 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:15.387 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:15.387 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "00e42188-16f6-11ef-bc9a-c7b262605968", 2024-05-20T22:13:15.388 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:15.389 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:13:15.390 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:13:15.391 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": false, 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:13:15.392 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:12:54.284834+0000", 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:13:15.393 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:15.841 INFO:teuthology.orchestra.run.smithi006.stderr:mgr not available, waiting (3/15)... 2024-05-20T22:13:16.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:15 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2072571143' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:13:16.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:15 smithi006 ceph-mon[24790]: mgrmap e3: smithi006.ssyogp(active, since 1.00814s) 2024-05-20T22:13:18.192 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:17 smithi006 ceph-mon[24790]: mgrmap e4: smithi006.ssyogp(active, since 2s) 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsid": "00e42188-16f6-11ef-bc9a-c7b262605968", 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "health": { 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "checks": {}, 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mutes": [] 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum": [ 2024-05-20T22:13:18.558 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 0 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_names": [ 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "smithi006" 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "quorum_age": 13, 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "monmap": { 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_mons": 1 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osdmap": { 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_osds": 0, 2024-05-20T22:13:18.559 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-05-20T22:13:18.560 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-05-20T22:13:18.563 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgmap": { 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_pools": 0, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_objects": 0, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-05-20T22:13:18.564 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "bytes_total": 0 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "fsmap": { 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "by_rank": [], 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "up:standby": 0 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap": { 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modules": [ 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "iostat", 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "nfs", 2024-05-20T22:13:18.565 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "restful" 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ], 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "servicemap": { 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 1, 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "modified": "2024-05-20T22:12:54.284834+0000", 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "services": {} 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: }, 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "progress_events": {} 2024-05-20T22:13:18.566 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:19.153 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:18 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/519886874' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-05-20T22:13:19.155 INFO:teuthology.orchestra.run.smithi006.stderr:mgr is available 2024-05-20T22:13:19.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:19.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [global] 2024-05-20T22:13:19.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: fsid = 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:13:19.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-05-20T22:13:19.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-05-20T22:13:19.738 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [mgr] 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: [osd] 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-05-20T22:13:19.739 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-05-20T22:13:20.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:19 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2009648287' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-05-20T22:13:20.304 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling cephadm module... 2024-05-20T22:13:21.776 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:21 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/827403433' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-05-20T22:13:22.693 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:22 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/827403433' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-05-20T22:13:22.693 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:22 smithi006 ceph-mon[24790]: mgrmap e5: smithi006.ssyogp(active, since 6s) 2024-05-20T22:13:22.694 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:22.694 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 5, 2024-05-20T22:13:22.694 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2024-05-20T22:13:22.694 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "active_name": "smithi006.ssyogp", 2024-05-20T22:13:22.694 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-20T22:13:22.694 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:23.304 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for the mgr to restart... 2024-05-20T22:13:23.304 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr epoch 5... 2024-05-20T22:13:23.612 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:23 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3870415388' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-20T22:13:28.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:27 smithi006 ceph-mon[24790]: Active manager daemon smithi006.ssyogp restarted 2024-05-20T22:13:28.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:27 smithi006 ceph-mon[24790]: Activating manager daemon smithi006.ssyogp 2024-05-20T22:13:28.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:27 smithi006 ceph-mon[24790]: osdmap e2: 0 total, 0 up, 0 in 2024-05-20T22:13:28.892 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:28.892 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-05-20T22:13:28.892 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "initialized": true 2024-05-20T22:13:28.892 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:29.199 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: mgrmap e6: smithi006.ssyogp(active, starting, since 0.052601s) 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:13:29.200 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:28 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:13:29.487 INFO:teuthology.orchestra.run.smithi006.stderr:mgr epoch 5 is available 2024-05-20T22:13:29.488 INFO:teuthology.orchestra.run.smithi006.stderr:Setting orchestrator backend to cephadm... 2024-05-20T22:13:30.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:29 smithi006 ceph-mon[24790]: mgrmap e7: smithi006.ssyogp(active, since 1.06036s) 2024-05-20T22:13:30.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:29 smithi006 ceph-mon[24790]: [20/May/2024:22:13:29] ENGINE Bus STARTING 2024-05-20T22:13:30.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:29 smithi006 ceph-mon[24790]: [20/May/2024:22:13:29] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:13:30.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:29 smithi006 ceph-mon[24790]: [20/May/2024:22:13:29] ENGINE Bus STARTED 2024-05-20T22:13:30.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:29 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:30.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:29 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:31.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:31 smithi006 ceph-mon[24790]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:31.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:31 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:31.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:31 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:31.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:31 smithi006 ceph-mon[24790]: mgrmap e8: smithi006.ssyogp(active, since 2s) 2024-05-20T22:13:31.258 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: value unchanged 2024-05-20T22:13:31.759 INFO:teuthology.orchestra.run.smithi006.stderr:Generating ssh key... 2024-05-20T22:13:32.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:32 smithi006 ceph-mon[24790]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:33.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:33 smithi006 ceph-mon[24790]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:33.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:33 smithi006 ceph-mon[24790]: Generating ssh key... 2024-05-20T22:13:33.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:33 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:33.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:33 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:34.125 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAoEAdkNPkXwO//9GtXVbKXhNyBNO+uTnHQaf1ajVaIDuaf1q8bQI0MIveV/vRuIGbwzcpK2KSAWhHFrN+6BUjT599pnTK7RfoVmD7lpctA/y0bsRGjjN3USdgy912iVy6x96gg71t0+CQ/FmA4et08krpbDjOOBcVJLa0fa4NTNCzrTRw40HUImpRwhY/8+KDLwhoGdy2RSjUlUiX8Ax6OksAT2kVD0wcVwxjaPhV6xKUIdL7Gb9jymCPpuFPFp3PpRlyZJdekjEN4DZoAt02eZGMCITfxPt3fEIv///KmSqNusy4XwqdhXV+1vrCxocumWl0lpHelA2IfG3Wgg2EPSowfn9ibGC3lWzitOVlL9QS+at2Uj35eCC6bcNtf0DQiyUz7D5c3pkS2V4rflF2O15hNNgj3oVh0gf5rzN6CpvkJT+ZRPIbiWJblPJ6RRFs4WS+wgfr73WYt5kmZjgyBdzGahlYSMA1BR4ukNyQY7lZP4qBv8g57K2bvWjNIA8= ceph-00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:13:34.713 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:34 smithi006 ceph-mon[24790]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:34.716 INFO:teuthology.orchestra.run.smithi006.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-05-20T22:13:34.716 INFO:teuthology.orchestra.run.smithi006.stderr:Adding key to root@localhost authorized_keys... 2024-05-20T22:13:34.717 INFO:teuthology.orchestra.run.smithi006.stderr:Adding host smithi006... 2024-05-20T22:13:36.236 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Added host 'smithi006' with addr '172.21.15.6' 2024-05-20T22:13:36.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:36 smithi006 ceph-mon[24790]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi006", "addr": "172.21.15.6", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:36.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:36 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:36.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:36 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:36.809 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying mon service with default placement... 2024-05-20T22:13:37.421 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled mon update... 2024-05-20T22:13:37.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:37 smithi006 ceph-mon[24790]: Deploying cephadm binary to smithi006 2024-05-20T22:13:37.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:37 smithi006 ceph-mon[24790]: Added host smithi006 2024-05-20T22:13:37.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:37 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:38.285 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying mgr service with default placement... 2024-05-20T22:13:38.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:38 smithi006 ceph-mon[24790]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:38.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:38 smithi006 ceph-mon[24790]: Saving service mon spec with placement count:5 2024-05-20T22:13:39.887 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-05-20T22:13:40.374 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying crash service with default placement... 2024-05-20T22:13:40.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:40 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:40.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:40 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:40.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:40 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:40.969 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled crash update... 2024-05-20T22:13:41.627 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:41 smithi006 ceph-mon[24790]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:41.628 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:41 smithi006 ceph-mon[24790]: Saving service mgr spec with placement count:2 2024-05-20T22:13:41.628 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:41 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:41.970 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying prometheus service with default placement... 2024-05-20T22:13:42.665 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-05-20T22:13:42.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:42 smithi006 ceph-mon[24790]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:42.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:42 smithi006 ceph-mon[24790]: Saving service crash spec with placement * 2024-05-20T22:13:42.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:42 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:42.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:42 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:43.178 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying grafana service with default placement... 2024-05-20T22:13:43.751 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:43 smithi006 ceph-mon[24790]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:43.752 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:43 smithi006 ceph-mon[24790]: Saving service prometheus spec with placement count:1 2024-05-20T22:13:43.752 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:43 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:43.764 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-05-20T22:13:44.420 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying node-exporter service with default placement... 2024-05-20T22:13:45.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:44 smithi006 ceph-mon[24790]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:45.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:44 smithi006 ceph-mon[24790]: Saving service grafana spec with placement count:1 2024-05-20T22:13:45.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:44 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:45.831 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-05-20T22:13:46.439 INFO:teuthology.orchestra.run.smithi006.stderr:Deploying alertmanager service with default placement... 2024-05-20T22:13:47.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:46 smithi006 ceph-mon[24790]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:47.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:46 smithi006 ceph-mon[24790]: Saving service node-exporter spec with placement * 2024-05-20T22:13:47.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:46 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:47.165 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-05-20T22:13:48.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:48 smithi006 ceph-mon[24790]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:13:48.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:48 smithi006 ceph-mon[24790]: Saving service alertmanager spec with placement count:1 2024-05-20T22:13:48.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:48 smithi006 ceph-mon[24790]: from='mgr.14120 172.21.15.6:0/3070401640' entity='mgr.smithi006.ssyogp' 2024-05-20T22:13:49.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:49 smithi006 ceph-mon[24790]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:13:49.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:49 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3399507004' entity='client.admin' 2024-05-20T22:13:50.399 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling the dashboard module... 2024-05-20T22:13:51.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:50 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2754123703' entity='client.admin' 2024-05-20T22:13:51.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:50 smithi006 ceph-mon[24790]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:13:52.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:51 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3545504844' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-05-20T22:13:53.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:52 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3545504844' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-05-20T22:13:53.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:52 smithi006 ceph-mon[24790]: mgrmap e9: smithi006.ssyogp(active, since 23s) 2024-05-20T22:13:53.075 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:53.075 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "epoch": 9, 2024-05-20T22:13:53.075 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "available": true, 2024-05-20T22:13:53.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "active_name": "smithi006.ssyogp", 2024-05-20T22:13:53.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "num_standby": 0 2024-05-20T22:13:53.076 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:53.646 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for the mgr to restart... 2024-05-20T22:13:53.646 INFO:teuthology.orchestra.run.smithi006.stderr:Waiting for mgr epoch 9... 2024-05-20T22:13:53.895 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:53 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3579118823' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: Active manager daemon smithi006.ssyogp restarted 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: Activating manager daemon smithi006.ssyogp 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: osdmap e3: 0 total, 0 up, 0 in 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: mgrmap e10: smithi006.ssyogp(active, starting, since 0.0532514s) 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:13:59.216 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:13:59.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:13:59.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:13:59.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:59.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:13:59.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:13:59.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:13:59 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:13:59.218 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: { 2024-05-20T22:13:59.218 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-05-20T22:13:59.219 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: "initialized": true 2024-05-20T22:13:59.219 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: } 2024-05-20T22:13:59.906 INFO:teuthology.orchestra.run.smithi006.stderr:mgr epoch 9 is available 2024-05-20T22:13:59.906 INFO:teuthology.orchestra.run.smithi006.stderr:Generating a dashboard self-signed certificate... 2024-05-20T22:14:00.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:00 smithi006 ceph-mon[24790]: mgrmap e11: smithi006.ssyogp(active, since 1.05606s) 2024-05-20T22:14:00.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:00 smithi006 ceph-mon[24790]: [20/May/2024:22:13:59] ENGINE Bus STARTING 2024-05-20T22:14:00.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:00 smithi006 ceph-mon[24790]: [20/May/2024:22:13:59] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:14:00.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:00 smithi006 ceph-mon[24790]: [20/May/2024:22:13:59] ENGINE Bus STARTED 2024-05-20T22:14:00.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:00 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:00.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:00 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:00.735 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: Self-signed certificate created 2024-05-20T22:14:01.388 INFO:teuthology.orchestra.run.smithi006.stderr:Creating initial admin user... 2024-05-20T22:14:01.997 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:01 smithi006 ceph-mon[24790]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:14:01.997 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:01 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:01.997 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:01 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:01.997 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:01 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:01.997 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:01 smithi006 ceph-mon[24790]: mgrmap e12: smithi006.ssyogp(active, since 2s) 2024-05-20T22:14:02.255 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$W7S4EzUtRL0HYghb80uXZO4/QmUXj5gJwN9dHjJlFhsgkQrzIvh9i", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1716243242, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-05-20T22:14:03.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:03 smithi006 ceph-mon[24790]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:14:03.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:03 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:03.400 INFO:teuthology.orchestra.run.smithi006.stderr:Fetching dashboard port number... 2024-05-20T22:14:04.016 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: 8443 2024-05-20T22:14:04.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:04 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3453622457' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-05-20T22:14:04.617 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: disabled 2024-05-20T22:14:04.624 INFO:teuthology.orchestra.run.smithi006.stderr:systemctl: inactive 2024-05-20T22:14:04.625 INFO:teuthology.orchestra.run.smithi006.stderr:firewalld.service is not enabled 2024-05-20T22:14:04.625 INFO:teuthology.orchestra.run.smithi006.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-05-20T22:14:04.627 INFO:teuthology.orchestra.run.smithi006.stderr:Ceph Dashboard is now available at: 2024-05-20T22:14:04.627 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:04.627 INFO:teuthology.orchestra.run.smithi006.stderr: URL: https://smithi006.front.sepia.ceph.com:8443/ 2024-05-20T22:14:04.627 INFO:teuthology.orchestra.run.smithi006.stderr: User: admin 2024-05-20T22:14:04.628 INFO:teuthology.orchestra.run.smithi006.stderr: Password: xubam1cg78 2024-05-20T22:14:04.628 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:04.628 INFO:teuthology.orchestra.run.smithi006.stderr:Enabling autotune for osd_memory_target 2024-05-20T22:14:05.519 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:05 smithi006 ceph-mon[24790]: mgrmap e13: smithi006.ssyogp(active, since 6s) 2024-05-20T22:14:06.518 INFO:teuthology.orchestra.run.smithi006.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-05-20T22:14:07.062 INFO:teuthology.orchestra.run.smithi006.stderr:You can access the Ceph CLI with: 2024-05-20T22:14:07.062 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr:Please consider enabling telemetry to help improve Ceph: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: ceph telemetry on 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr:For more information see: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:14:07.063 INFO:teuthology.orchestra.run.smithi006.stderr:Bootstrap complete. 2024-05-20T22:14:07.098 INFO:tasks.cephadm:Fetching config... 2024-05-20T22:14:07.099 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:14:07.099 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-05-20T22:14:07.117 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-05-20T22:14:07.117 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:14:07.117 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-05-20T22:14:07.174 INFO:tasks.cephadm:Fetching mon keyring... 2024-05-20T22:14:07.175 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:14:07.175 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/keyring of=/dev/stdout 2024-05-20T22:14:07.243 INFO:tasks.cephadm:Fetching pub ssh key... 2024-05-20T22:14:07.244 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:14:07.244 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-05-20T22:14:07.300 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-05-20T22:14:07.300 DEBUG:teuthology.orchestra.run.smithi006:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAoEAdkNPkXwO//9GtXVbKXhNyBNO+uTnHQaf1ajVaIDuaf1q8bQI0MIveV/vRuIGbwzcpK2KSAWhHFrN+6BUjT599pnTK7RfoVmD7lpctA/y0bsRGjjN3USdgy912iVy6x96gg71t0+CQ/FmA4et08krpbDjOOBcVJLa0fa4NTNCzrTRw40HUImpRwhY/8+KDLwhoGdy2RSjUlUiX8Ax6OksAT2kVD0wcVwxjaPhV6xKUIdL7Gb9jymCPpuFPFp3PpRlyZJdekjEN4DZoAt02eZGMCITfxPt3fEIv///KmSqNusy4XwqdhXV+1vrCxocumWl0lpHelA2IfG3Wgg2EPSowfn9ibGC3lWzitOVlL9QS+at2Uj35eCC6bcNtf0DQiyUz7D5c3pkS2V4rflF2O15hNNgj3oVh0gf5rzN6CpvkJT+ZRPIbiWJblPJ6RRFs4WS+wgfr73WYt5kmZjgyBdzGahlYSMA1BR4ukNyQY7lZP4qBv8g57K2bvWjNIA8= ceph-00e42188-16f6-11ef-bc9a-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-20T22:14:07.382 INFO:teuthology.orchestra.run.smithi006.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAoEAdkNPkXwO//9GtXVbKXhNyBNO+uTnHQaf1ajVaIDuaf1q8bQI0MIveV/vRuIGbwzcpK2KSAWhHFrN+6BUjT599pnTK7RfoVmD7lpctA/y0bsRGjjN3USdgy912iVy6x96gg71t0+CQ/FmA4et08krpbDjOOBcVJLa0fa4NTNCzrTRw40HUImpRwhY/8+KDLwhoGdy2RSjUlUiX8Ax6OksAT2kVD0wcVwxjaPhV6xKUIdL7Gb9jymCPpuFPFp3PpRlyZJdekjEN4DZoAt02eZGMCITfxPt3fEIv///KmSqNusy4XwqdhXV+1vrCxocumWl0lpHelA2IfG3Wgg2EPSowfn9ibGC3lWzitOVlL9QS+at2Uj35eCC6bcNtf0DQiyUz7D5c3pkS2V4rflF2O15hNNgj3oVh0gf5rzN6CpvkJT+ZRPIbiWJblPJ6RRFs4WS+wgfr73WYt5kmZjgyBdzGahlYSMA1BR4ukNyQY7lZP4qBv8g57K2bvWjNIA8= ceph-00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:14:07.394 DEBUG:teuthology.orchestra.run.smithi078:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAoEAdkNPkXwO//9GtXVbKXhNyBNO+uTnHQaf1ajVaIDuaf1q8bQI0MIveV/vRuIGbwzcpK2KSAWhHFrN+6BUjT599pnTK7RfoVmD7lpctA/y0bsRGjjN3USdgy912iVy6x96gg71t0+CQ/FmA4et08krpbDjOOBcVJLa0fa4NTNCzrTRw40HUImpRwhY/8+KDLwhoGdy2RSjUlUiX8Ax6OksAT2kVD0wcVwxjaPhV6xKUIdL7Gb9jymCPpuFPFp3PpRlyZJdekjEN4DZoAt02eZGMCITfxPt3fEIv///KmSqNusy4XwqdhXV+1vrCxocumWl0lpHelA2IfG3Wgg2EPSowfn9ibGC3lWzitOVlL9QS+at2Uj35eCC6bcNtf0DQiyUz7D5c3pkS2V4rflF2O15hNNgj3oVh0gf5rzN6CpvkJT+ZRPIbiWJblPJ6RRFs4WS+wgfr73WYt5kmZjgyBdzGahlYSMA1BR4ukNyQY7lZP4qBv8g57K2bvWjNIA8= ceph-00e42188-16f6-11ef-bc9a-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-05-20T22:14:07.434 INFO:teuthology.orchestra.run.smithi078.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAoEAdkNPkXwO//9GtXVbKXhNyBNO+uTnHQaf1ajVaIDuaf1q8bQI0MIveV/vRuIGbwzcpK2KSAWhHFrN+6BUjT599pnTK7RfoVmD7lpctA/y0bsRGjjN3USdgy912iVy6x96gg71t0+CQ/FmA4et08krpbDjOOBcVJLa0fa4NTNCzrTRw40HUImpRwhY/8+KDLwhoGdy2RSjUlUiX8Ax6OksAT2kVD0wcVwxjaPhV6xKUIdL7Gb9jymCPpuFPFp3PpRlyZJdekjEN4DZoAt02eZGMCITfxPt3fEIv///KmSqNusy4XwqdhXV+1vrCxocumWl0lpHelA2IfG3Wgg2EPSowfn9ibGC3lWzitOVlL9QS+at2Uj35eCC6bcNtf0DQiyUz7D5c3pkS2V4rflF2O15hNNgj3oVh0gf5rzN6CpvkJT+ZRPIbiWJblPJ6RRFs4WS+wgfr73WYt5kmZjgyBdzGahlYSMA1BR4ukNyQY7lZP4qBv8g57K2bvWjNIA8= ceph-00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:14:07.446 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-05-20T22:14:07.766 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:07 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/417991723' entity='client.admin' 2024-05-20T22:14:11.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:10 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1807254012' entity='client.admin' 2024-05-20T22:14:11.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:10 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:11.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:10 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi006", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:14:11.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:10 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:11.081 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-05-20T22:14:11.082 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-05-20T22:14:11.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:11 smithi006 ceph-mon[24790]: Deploying daemon alertmanager.smithi006 on smithi006 2024-05-20T22:14:14.582 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi078 2024-05-20T22:14:14.582 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-20T22:14:14.582 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/etc/ceph/ceph.conf 2024-05-20T22:14:14.601 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-20T22:14:14.601 DEBUG:teuthology.orchestra.run.smithi078:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:14:14.659 INFO:tasks.cephadm:Adding host smithi078 to orchestrator... 2024-05-20T22:14:14.660 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch host add smithi078 2024-05-20T22:14:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:14 smithi006 ceph-mon[24790]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:14:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:14 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:19.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:19 smithi006 ceph-mon[24790]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:20.300 INFO:teuthology.orchestra.run.smithi006.stdout:Added host 'smithi078' with addr '172.21.15.78' 2024-05-20T22:14:20.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:20 smithi006 ceph-mon[24790]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:14:20.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:20 smithi006 ceph-mon[24790]: Deploying cephadm binary to smithi078 2024-05-20T22:14:20.955 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch host ls --format=json 2024-05-20T22:14:21.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:21 smithi006 ceph-mon[24790]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:21.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:21 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:21.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:21 smithi006 ceph-mon[24790]: Added host smithi078 2024-05-20T22:14:22.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:22 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:22 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:14:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:22 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-20T22:14:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:22 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:14:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:22 smithi006 ceph-mon[24790]: Deploying daemon crash.smithi006 on smithi006 2024-05-20T22:14:24.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:23 smithi006 ceph-mon[24790]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:24.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:23 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:24.670 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:14:24.670 INFO:teuthology.orchestra.run.smithi006.stdout:[{"addr": "172.21.15.6", "hostname": "smithi006", "labels": [], "status": ""}, {"addr": "172.21.15.78", "hostname": "smithi078", "labels": [], "status": ""}] 2024-05-20T22:14:25.288 INFO:tasks.cephadm:Setting crush tunables to default 2024-05-20T22:14:25.288 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd crush tunables default 2024-05-20T22:14:25.871 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:25 smithi006 ceph-mon[24790]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:25.872 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:25 smithi006 ceph-mon[24790]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:14:25.872 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:25 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:25.872 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:25 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:25.872 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:25 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:25.872 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:25 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:14:25.872 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:25 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:26.821 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:26 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:14:26.821 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:26 smithi006 ceph-mon[24790]: Deploying daemon grafana.smithi006 on smithi006 2024-05-20T22:14:27.748 INFO:teuthology.orchestra.run.smithi006.stderr:adjusted tunables profile to default 2024-05-20T22:14:28.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:27 smithi006 ceph-mon[24790]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:28.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:27 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/4173600042' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-20T22:14:28.455 INFO:tasks.cephadm:Adding mon.smithi006 on smithi006 2024-05-20T22:14:28.455 INFO:tasks.cephadm:Adding mon.smithi078 on smithi078 2024-05-20T22:14:28.455 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch apply mon '2;smithi006:172.21.15.6=smithi006;smithi078:172.21.15.78=smithi078' 2024-05-20T22:14:29.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:28 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/4173600042' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-20T22:14:29.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:28 smithi006 ceph-mon[24790]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:14:29.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:28 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:30.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:29 smithi006 ceph-mon[24790]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:30.119 INFO:teuthology.orchestra.run.smithi078.stdout:Scheduled mon update... 2024-05-20T22:14:30.861 DEBUG:teuthology.orchestra.run.smithi078:mon.smithi078> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi078.service 2024-05-20T22:14:30.863 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:30.864 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:31.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:31 smithi006 ceph-mon[24790]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi006:172.21.15.6=smithi006;smithi078:172.21.15.78=smithi078", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:14:31.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:31 smithi006 ceph-mon[24790]: Saving service mon spec with placement smithi006:172.21.15.6=smithi006;smithi078:172.21.15.78=smithi078;count:2 2024-05-20T22:14:31.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:31 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:31.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:31 smithi006 ceph-mon[24790]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:32.899 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:32.899 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:32.901 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:33.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:33 smithi006 ceph-mon[24790]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:33.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:33 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/887139330' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:34.377 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:34.377 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:35.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:35 smithi006 ceph-mon[24790]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:36.189 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:36.189 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:36.191 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:36.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:36 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/4028638116' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:37.723 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:37.723 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:37.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:37 smithi006 ceph-mon[24790]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:39.532 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:39.532 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:39.534 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:39.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:39 smithi006 ceph-mon[24790]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:40.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:40 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/898379053' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:41.029 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:41.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:41.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:41 smithi006 ceph-mon[24790]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:42.866 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:42.866 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:42.868 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:44.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:43 smithi006 ceph-mon[24790]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:44.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:43 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:44.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:43 smithi006 ceph-mon[24790]: Deploying daemon node-exporter.smithi006 on smithi006 2024-05-20T22:14:44.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:43 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/2338077406' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:44.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:43 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:44.398 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:44.398 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:46.208 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:46.208 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:46.210 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:46.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:45 smithi006 ceph-mon[24790]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:47.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:46 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/3330223443' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:47.709 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:47.709 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:48.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:47 smithi006 ceph-mon[24790]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:14:48.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:47 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:48.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:47 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-20T22:14:49.485 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:49.485 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:49.487 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:49.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:49 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:49.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:49 smithi006 ceph-mon[24790]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-20T22:14:49.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:49 smithi006 ceph-mon[24790]: mgrmap e14: smithi006.ssyogp(active, since 50s) 2024-05-20T22:14:50.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:50 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/782829983' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:51.020 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:51.021 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:52.770 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:52.770 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:52.772 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:53.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:52 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/3511703467' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:54.350 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:54.351 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:55.166 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:54 smithi006 ceph-mon[24790]: Active manager daemon smithi006.ssyogp restarted 2024-05-20T22:14:55.166 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:54 smithi006 ceph-mon[24790]: Activating manager daemon smithi006.ssyogp 2024-05-20T22:14:55.166 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:54 smithi006 ceph-mon[24790]: osdmap e5: 0 total, 0 up, 0 in 2024-05-20T22:14:56.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: mgrmap e15: smithi006.ssyogp(active, starting, since 0.056583s) 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:14:56.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:14:56.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: [20/May/2024:22:14:55] ENGINE Bus STARTING 2024-05-20T22:14:56.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: [20/May/2024:22:14:55] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:14:56.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: [20/May/2024:22:14:55] ENGINE Bus STARTED 2024-05-20T22:14:56.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:56.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:56.295 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:14:56.295 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:14:56.297 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:14:57.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:56 smithi006 ceph-mon[24790]: mgrmap e16: smithi006.ssyogp(active, since 1.06628s) 2024-05-20T22:14:57.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:56 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:57.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:56 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/2158602420' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:14:57.831 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:14:57.831 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:14:58.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:57 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:14:58.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:14:57 smithi006 ceph-mon[24790]: mgrmap e17: smithi006.ssyogp(active, since 2s) 2024-05-20T22:15:00.523 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:15:00.523 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:15:00.525 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:15:01.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:01.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:01.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:15:01.393 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:01 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/558592634' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:02.242 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:15:02.242 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:15:03.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:03.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:03.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi006", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:15:03.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:03.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:03.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:15:03.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-20T22:15:03.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:04.104 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:15:04.104 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:15:04.106 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:15:04.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:04 smithi006 ceph-mon[24790]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:15:04.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:04 smithi006 ceph-mon[24790]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:15:04.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:04 smithi006 ceph-mon[24790]: Deploying daemon crash.smithi078 on smithi078 2024-05-20T22:15:04.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:04 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/1248551963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:05.666 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:15:05.666 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:15:07.639 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:15:07.639 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:15:07.640 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:15:08.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:07 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/1247715036' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:09.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:09.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:15:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T22:15:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:15:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:08 smithi006 ceph-mon[24790]: Deploying daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:15:09.414 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:15:09.414 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:15:11.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:10 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:11.386 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:15:11.386 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:15:11.388 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:15:12.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:11 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/2837847157' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:13.027 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:15:13.028 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:15:13.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:13.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:15:13.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:13.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:13 smithi006 ceph-mon[24790]: Deploying daemon mon.smithi078 on smithi078 2024-05-20T22:15:15.886 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:15:15.888 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":1,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:12:48.663636Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-05-20T22:15:15.888 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 1 2024-05-20T22:15:16.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:15 smithi006 ceph-mon[24790]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:16.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:15 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:16.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:15 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/4087777823' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:16.503 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:16 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:15:17.051 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:16 smithi078 podman[27204]: 2024-05-20 22:15:16.881246744 +0000 UTC m=+0.018109977 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-05-20T22:15:17.052 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:16 smithi078 podman[27204]: 2024-05-20 22:15:16.985465238 +0000 UTC m=+0.122328455 container create e4bbcbbe002698380a4c0867a8d6cdb8c9a4f0a8eec13012abbd8f3582f1de12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, RELEASE=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870) 2024-05-20T22:15:17.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 podman[27204]: 2024-05-20 22:15:17.213558395 +0000 UTC m=+0.350421635 container init e4bbcbbe002698380a4c0867a8d6cdb8c9a4f0a8eec13012abbd8f3582f1de12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, io.openshift.expose-services=, com.redhat.component=centos-stream-container, version=8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.k8s.display-name=CentOS Stream 8, ceph=True, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754, architecture=x86_64) 2024-05-20T22:15:17.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 podman[27204]: 2024-05-20 22:15:17.216983699 +0000 UTC m=+0.353846933 container start e4bbcbbe002698380a4c0867a8d6cdb8c9a4f0a8eec13012abbd8f3582f1de12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, ceph=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vendor=Red Hat, Inc., architecture=x86_64, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream) 2024-05-20T22:15:17.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:15:17.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-05-20T22:15:17.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pidfile_write: ignore empty --pid-file 2024-05-20T22:15:17.340 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: load: jerasure load: lrc 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: RocksDB version: 6.15.5 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Compile date Apr 18 2022 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: DB SUMMARY 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: DB Session ID: 9ZVH6KRR4ZXURSBP9KQG 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 0, files: 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000004.log size: 511 ; 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:15:17.341 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.env: 0x55d74fcc6860 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.fs: Posix File System 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.info_log: 0x55d7516dffa0 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.statistics: (nil) 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.use_fsync: 0 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:15:17.342 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.db_log_dir: 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi078/store.db 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:15:17.343 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:15:17.345 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.write_buffer_manager: 0x55d7517d0330 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:15:17.346 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.unordered_write: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.row_cache: None 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.wal_filter: None 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.preserve_deletes: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:15:17.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:15:17.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_open_files: -1 2024-05-20T22:15:17.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Compression algorithms supported: 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kZSTD supported: 0 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:15:17.350 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:15:17.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:15:17.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:15:17.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:15:17.351 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000003 2024-05-20T22:15:17.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-05-20T22:15:17.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:15:17.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.merge_operator: 2024-05-20T22:15:17.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_filter: None 2024-05-20T22:15:17.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:15:17.352 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d7516abd00) 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-05-20T22:15:17.353 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: hash_index_allow_collision: 1 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 1 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x55d751716170 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:15:17.354 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 4 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-05-20T22:15:17.355 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression: NoCompression 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.num_levels: 7 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:15:17.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:15:17.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:15:17.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.arena_block_size: 4194304 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:15:17.359 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:15:17.360 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.ttl: 2592000 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:15:17.361 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243317243724, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243317244142, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1716243317, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "85be3911-958d-4b3a-b8af-a893891b4437", "db_session_id": "9ZVH6KRR4ZXURSBP9KQG"}} 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243317244545, "job": 1, "event": "recovery_finished"} 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55d7516fca80 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: DB pointer 0x55d75170c000 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-05-20T22:15:17.362 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.7 0.00 0.00 1 0.000 0 0 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.7 0.00 0.00 1 0.000 0 0 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.7 0.00 0.00 1 0.000 0 0 2024-05-20T22:15:17.363 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.7 0.00 0.00 1 0.000 0 0 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:15:17.364 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.7 0.00 0.00 1 0.000 0 0 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.7 0.00 0.00 1 0.000 0 0 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-05-20T22:15:17.365 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.7 0.00 0.00 1 0.000 0 0 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:15:17.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:15:17.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:15:17.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:15:17.370 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078 does not exist in monmap, will attempt to join an existing cluster 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: using public_addr v2:172.21.15.78:0/0 -> [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: starting mon.smithi078 rank -1 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(???) e0 preinit fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).mds e1 new map 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).mds e1 print_map 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e1 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: -1 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: No filesystems configured 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-05-20T22:15:17.371 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi078", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:15:17.372 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Deploying cephadm binary to smithi078 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Added host smithi078 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Deploying daemon crash.smithi006 on smithi006 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:15:17.373 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Deploying daemon grafana.smithi006 on smithi006 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/4173600042' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/4173600042' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: osdmap e4: 0 total, 0 up, 0 in 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi006:172.21.15.6=smithi006;smithi078:172.21.15.78=smithi078", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Saving service mon spec with placement smithi006:172.21.15.6=smithi006;smithi078:172.21.15.78=smithi078;count:2 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/887139330' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/4028638116' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.375 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/898379053' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Deploying daemon node-exporter.smithi006 on smithi006 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/2338077406' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.376 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/3330223443' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14162 172.21.15.6:0/3038494853' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mgrmap e14: smithi006.ssyogp(active, since 50s) 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/782829983' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/3511703467' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Active manager daemon smithi006.ssyogp restarted 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Activating manager daemon smithi006.ssyogp 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: osdmap e5: 0 total, 0 up, 0 in 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mgrmap e15: smithi006.ssyogp(active, starting, since 0.056583s) 2024-05-20T22:15:17.377 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:15:17.378 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: [20/May/2024:22:14:55] ENGINE Bus STARTING 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: [20/May/2024:22:14:55] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: [20/May/2024:22:14:55] ENGINE Bus STARTED 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mgrmap e16: smithi006.ssyogp(active, since 1.06628s) 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/2158602420' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mgrmap e17: smithi006.ssyogp(active, since 2s) 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi078", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/558592634' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.379 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi006", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Deploying daemon crash.smithi078 on smithi078 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/1248551963' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/1247715036' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:15:17.380 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Deploying daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/2837847157' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: Deploying daemon mon.smithi078 on smithi078 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/4087777823' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:17.381 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-05-20T22:15:17.382 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 ceph-mon[27225]: mon.smithi078@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-05-20T22:15:17.382 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 bash[27204]: e4bbcbbe002698380a4c0867a8d6cdb8c9a4f0a8eec13012abbd8f3582f1de12 2024-05-20T22:15:17.382 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:17 smithi078 systemd[1]: Started Ceph mon.smithi078 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:15:17.818 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-05-20T22:15:17.818 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mon dump -f json 2024-05-20T22:15:22.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:22.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:15:22.603 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: mon.smithi006 calling monitor election 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: mon.smithi078 calling monitor election 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: mon.smithi006 is new leader, mons smithi006,smithi078 in quorum (ranks 0,1) 2024-05-20T22:15:22.604 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: monmap e2: 2 mons at {smithi006=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: fsmap 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: osdmap e5: 0 total, 0 up, 0 in 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: mgrmap e17: smithi006.ssyogp(active, since 27s) 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: overall HEALTH_OK 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:15:22.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:22 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:22.642 INFO:teuthology.orchestra.run.smithi078.stdout: 2024-05-20T22:15:22.642 INFO:teuthology.orchestra.run.smithi078.stdout:{"epoch":2,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","modified":"2024-05-20T22:15:17.269330Z","created":"2024-05-20T22:12:48.663636Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi006","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:3300","nonce":0},{"type":"v1","addr":"172.21.15.6:6789","nonce":0}]},"addr":"172.21.15.6:6789/0","public_addr":"172.21.15.6:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi078","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:3300","nonce":0},{"type":"v1","addr":"172.21.15.78:6789","nonce":0}]},"addr":"172.21.15.78:6789/0","public_addr":"172.21.15.78:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-05-20T22:15:22.644 INFO:teuthology.orchestra.run.smithi078.stderr:dumped monmap epoch 2 2024-05-20T22:15:22.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: mon.smithi006 calling monitor election 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: mon.smithi078 calling monitor election 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: mon.smithi006 is new leader, mons smithi006,smithi078 in quorum (ranks 0,1) 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: monmap e2: 2 mons at {smithi006=[v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0],smithi078=[v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0]} 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: fsmap 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: osdmap e5: 0 total, 0 up, 0 in 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: mgrmap e17: smithi006.ssyogp(active, since 27s) 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: overall HEALTH_OK 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/3843133598' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:15:22.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:22 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:23.316 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-05-20T22:15:23.316 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph config generate-minimal-conf 2024-05-20T22:15:23.643 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:23 smithi006 ceph-mon[24790]: Deploying daemon node-exporter.smithi078 on smithi078 2024-05-20T22:15:23.643 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:23 smithi006 ceph-mon[24790]: mgrmap e18: smithi006.ssyogp(active, since 27s), standbys: smithi078.vxzcbt 2024-05-20T22:15:23.644 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:23 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:15:23.644 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:23 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/1772529628' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:23.644 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:23 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:23.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:23 smithi078 ceph-mon[27225]: Deploying daemon node-exporter.smithi078 on smithi078 2024-05-20T22:15:23.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:23 smithi078 ceph-mon[27225]: mgrmap e18: smithi006.ssyogp(active, since 27s), standbys: smithi078.vxzcbt 2024-05-20T22:15:23.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:23 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:15:23.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:23 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/1772529628' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-05-20T22:15:23.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:23 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:15:24.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:24 smithi078 ceph-mon[27225]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:24.707 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:24 smithi006 ceph-mon[24790]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:25.382 INFO:teuthology.orchestra.run.smithi006.stdout:# minimal ceph.conf for 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:15:25.382 INFO:teuthology.orchestra.run.smithi006.stdout:[global] 2024-05-20T22:15:25.382 INFO:teuthology.orchestra.run.smithi006.stdout: fsid = 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:15:25.383 INFO:teuthology.orchestra.run.smithi006.stdout: mon_host = [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] 2024-05-20T22:15:26.088 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-05-20T22:15:26.088 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:15:26.088 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.conf 2024-05-20T22:15:26.116 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:15:26.117 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:15:26.121 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:25 smithi078 ceph-mon[27225]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:26.121 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:26.121 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:25 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2557761836' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:26.182 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-20T22:15:26.182 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.conf 2024-05-20T22:15:26.209 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-20T22:15:26.209 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:15:26.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:25 smithi006 ceph-mon[24790]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:26.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:26.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:25 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2557761836' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:26.274 INFO:tasks.cephadm:Deploying OSDs... 2024-05-20T22:15:26.275 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:15:26.275 DEBUG:teuthology.orchestra.run.smithi006:> dd if=/scratch_devs of=/dev/stdout 2024-05-20T22:15:26.291 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-20T22:15:26.292 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_1 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-05-20 22:14:43.476093961 +0000 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-05-20 22:11:30.887567650 +0000 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-05-20 22:11:30.887567650 +0000 2024-05-20T22:15:26.348 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-05-20 22:11:30.887567650 +0000 2024-05-20T22:15:26.349 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-20T22:15:26.416 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-05-20T22:15:26.417 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-05-20T22:15:26.417 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000270058 s, 1.9 MB/s 2024-05-20T22:15:26.418 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-20T22:15:26.476 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_2 2024-05-20T22:15:26.534 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-20T22:15:26.534 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:26.534 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-05-20T22:15:26.534 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:26.535 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:26.535 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-05-20 22:14:43.476093961 +0000 2024-05-20T22:15:26.535 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-05-20 22:11:31.229563783 +0000 2024-05-20T22:15:26.535 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-05-20 22:11:31.229563783 +0000 2024-05-20T22:15:26.535 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-05-20 22:11:31.229563783 +0000 2024-05-20T22:15:26.535 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-20T22:15:26.600 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-05-20T22:15:26.601 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-05-20T22:15:26.601 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000227024 s, 2.3 MB/s 2024-05-20T22:15:26.602 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-20T22:15:26.661 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_3 2024-05-20T22:15:26.716 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-05-20 22:14:43.477093949 +0000 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-05-20 22:11:31.588559724 +0000 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-05-20 22:11:31.588559724 +0000 2024-05-20T22:15:26.717 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-05-20 22:11:31.588559724 +0000 2024-05-20T22:15:26.717 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-20T22:15:26.781 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-05-20T22:15:26.781 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-05-20T22:15:26.781 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000205698 s, 2.5 MB/s 2024-05-20T22:15:26.782 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-20T22:15:26.840 DEBUG:teuthology.orchestra.run.smithi006:> stat /dev/vg_nvme/lv_4 2024-05-20T22:15:26.896 INFO:teuthology.orchestra.run.smithi006.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout:Access: 2024-05-20 22:14:43.477093949 +0000 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout:Modify: 2024-05-20 22:11:31.932555835 +0000 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout:Change: 2024-05-20 22:11:31.932555835 +0000 2024-05-20T22:15:26.897 INFO:teuthology.orchestra.run.smithi006.stdout: Birth: 2024-05-20 22:11:31.932555835 +0000 2024-05-20T22:15:26.897 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-20T22:15:26.961 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records in 2024-05-20T22:15:26.961 INFO:teuthology.orchestra.run.smithi006.stderr:1+0 records out 2024-05-20T22:15:26.961 INFO:teuthology.orchestra.run.smithi006.stderr:512 bytes copied, 0.000227512 s, 2.3 MB/s 2024-05-20T22:15:26.962 DEBUG:teuthology.orchestra.run.smithi006:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-20T22:15:27.019 DEBUG:teuthology.orchestra.run.smithi078:> set -ex 2024-05-20T22:15:27.019 DEBUG:teuthology.orchestra.run.smithi078:> dd if=/scratch_devs of=/dev/stdout 2024-05-20T22:15:27.038 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-05-20T22:15:27.038 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_1 2024-05-20T22:15:27.096 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-20 22:15:22.896758632 +0000 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-20 22:11:28.204695118 +0000 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-20 22:11:28.204695118 +0000 2024-05-20T22:15:27.097 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-20 22:11:28.204695118 +0000 2024-05-20T22:15:27.097 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-05-20T22:15:27.163 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-20T22:15:27.164 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-20T22:15:27.164 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000228197 s, 2.2 MB/s 2024-05-20T22:15:27.165 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-05-20T22:15:27.224 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_2 2024-05-20T22:15:27.281 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-05-20T22:15:27.281 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:27.281 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-05-20T22:15:27.281 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:27.281 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:27.281 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-20 22:15:22.896758632 +0000 2024-05-20T22:15:27.282 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-20 22:11:28.533689929 +0000 2024-05-20T22:15:27.282 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-20 22:11:28.533689929 +0000 2024-05-20T22:15:27.282 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-20 22:11:28.533689929 +0000 2024-05-20T22:15:27.282 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-05-20T22:15:27.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:27 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:27.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:27 smithi006 ceph-mon[24790]: Deploying daemon prometheus.smithi006 on smithi006 2024-05-20T22:15:27.346 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-20T22:15:27.346 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-20T22:15:27.346 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000220818 s, 2.3 MB/s 2024-05-20T22:15:27.347 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-05-20T22:15:27.404 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_3 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-20 22:15:22.896758632 +0000 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-20 22:11:28.880684456 +0000 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-20 22:11:28.880684456 +0000 2024-05-20T22:15:27.460 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-20 22:11:28.880684456 +0000 2024-05-20T22:15:27.461 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-05-20T22:15:27.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:27 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:27.521 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:27 smithi078 ceph-mon[27225]: Deploying daemon prometheus.smithi006 on smithi006 2024-05-20T22:15:27.524 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-20T22:15:27.524 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-20T22:15:27.524 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000190298 s, 2.7 MB/s 2024-05-20T22:15:27.525 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-05-20T22:15:27.582 DEBUG:teuthology.orchestra.run.smithi078:> stat /dev/vg_nvme/lv_4 2024-05-20T22:15:27.637 INFO:teuthology.orchestra.run.smithi078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-05-20T22:15:27.637 INFO:teuthology.orchestra.run.smithi078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-05-20T22:15:27.637 INFO:teuthology.orchestra.run.smithi078.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-05-20T22:15:27.637 INFO:teuthology.orchestra.run.smithi078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-05-20T22:15:27.638 INFO:teuthology.orchestra.run.smithi078.stdout:Context: system_u:object_r:device_t:s0 2024-05-20T22:15:27.638 INFO:teuthology.orchestra.run.smithi078.stdout:Access: 2024-05-20 22:15:22.897758617 +0000 2024-05-20T22:15:27.638 INFO:teuthology.orchestra.run.smithi078.stdout:Modify: 2024-05-20 22:11:29.225679015 +0000 2024-05-20T22:15:27.638 INFO:teuthology.orchestra.run.smithi078.stdout:Change: 2024-05-20 22:11:29.225679015 +0000 2024-05-20T22:15:27.638 INFO:teuthology.orchestra.run.smithi078.stdout: Birth: 2024-05-20 22:11:29.225679015 +0000 2024-05-20T22:15:27.638 DEBUG:teuthology.orchestra.run.smithi078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-05-20T22:15:27.701 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records in 2024-05-20T22:15:27.701 INFO:teuthology.orchestra.run.smithi078.stderr:1+0 records out 2024-05-20T22:15:27.701 INFO:teuthology.orchestra.run.smithi078.stderr:512 bytes copied, 0.000198224 s, 2.6 MB/s 2024-05-20T22:15:27.702 DEBUG:teuthology.orchestra.run.smithi078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-05-20T22:15:27.758 INFO:tasks.cephadm:Deploying osd.0 on smithi006 with /dev/vg_nvme/lv_4... 2024-05-20T22:15:27.759 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-20T22:15:28.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:28 smithi006 ceph-mon[24790]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:28.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:28 smithi078 ceph-mon[27225]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:30.380 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-20T22:15:30.380 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-20T22:15:30.380 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2024-05-20T22:15:30.380 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2024-05-20T22:15:30.381 INFO:teuthology.orchestra.run.smithi006.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0155712 s, 673 MB/s 2024-05-20T22:15:30.381 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2024-05-20T22:15:30.381 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:30 smithi006 ceph-mon[24790]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:30.381 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:30 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:30.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:30 smithi078 ceph-mon[27225]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:30.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:30 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:32.568 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_4 2024-05-20T22:15:32.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:32 smithi006 ceph-mon[24790]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:32.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:32 smithi078 ceph-mon[27225]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:34.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:34 smithi006 ceph-mon[24790]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:34 smithi078 ceph-mon[27225]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:36.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:36 smithi006 ceph-mon[24790]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:36.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:15:36.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:15:36.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:36.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:36 smithi078 ceph-mon[27225]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:36.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:15:36.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:15:36.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:37.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:37 smithi006 ceph-mon[24790]: from='client.14242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:15:37.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:37 smithi078 ceph-mon[27225]: from='client.14242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:15:38.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:38 smithi006 ceph-mon[24790]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:38.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:38 smithi078 ceph-mon[27225]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:40.624 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:40 smithi006 ceph-mon[24790]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:40.624 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:40.624 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:15:40.624 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:40.624 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:15:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:40 smithi078 ceph-mon[27225]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:15:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:40.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1195691658' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85a3ac26-fd4b-4838-8c9a-f936c3a4bc59"}]: dispatch 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1195691658' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85a3ac26-fd4b-4838-8c9a-f936c3a4bc59"}]': finished 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1948792358' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:15:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1195691658' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "85a3ac26-fd4b-4838-8c9a-f936c3a4bc59"}]: dispatch 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1195691658' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "85a3ac26-fd4b-4838-8c9a-f936c3a4bc59"}]': finished 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: osdmap e6: 1 total, 0 up, 1 in 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1948792358' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:15:42.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:43.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:43 smithi006 ceph-mon[24790]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:15:43.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:43 smithi006 ceph-mon[24790]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:15:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:43 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:15:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:43 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:15:44.725 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:44 smithi006 ceph-mon[24790]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:44.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:44 smithi078 ceph-mon[27225]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:46.236 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:46.236 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.236 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:15:46.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:15:46.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:15:46.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:46.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:46.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:15:46.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:46.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:15:46.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:15:46.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:47.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:46 smithi006 ceph-mon[24790]: Reconfiguring mon.smithi006 (unknown last config time)... 2024-05-20T22:15:47.252 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:46 smithi006 ceph-mon[24790]: Reconfiguring daemon mon.smithi006 on smithi006 2024-05-20T22:15:47.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:46 smithi078 ceph-mon[27225]: Reconfiguring mon.smithi006 (unknown last config time)... 2024-05-20T22:15:47.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:46 smithi078 ceph-mon[27225]: Reconfiguring daemon mon.smithi006 on smithi006 2024-05-20T22:15:48.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:47 smithi006 ceph-mon[24790]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:48.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:48.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:15:48.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:15:48.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:47 smithi078 ceph-mon[27225]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:15:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:15:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:49.192 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:48 smithi006 ceph-mon[24790]: Reconfiguring mgr.smithi006.ssyogp (unknown last config time)... 2024-05-20T22:15:49.192 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:48 smithi006 ceph-mon[24790]: Reconfiguring daemon mgr.smithi006.ssyogp on smithi006 2024-05-20T22:15:49.192 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:48 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:15:49.193 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:48 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:49.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:48 smithi078 ceph-mon[27225]: Reconfiguring mgr.smithi006.ssyogp (unknown last config time)... 2024-05-20T22:15:49.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:48 smithi078 ceph-mon[27225]: Reconfiguring daemon mgr.smithi006.ssyogp on smithi006 2024-05-20T22:15:49.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:48 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:15:49.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:48 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:15:50.234 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:49 smithi006 ceph-mon[24790]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:50.234 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:49 smithi006 ceph-mon[24790]: Deploying daemon osd.0 on smithi006 2024-05-20T22:15:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:49 smithi078 ceph-mon[27225]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:49 smithi078 ceph-mon[27225]: Deploying daemon osd.0 on smithi006 2024-05-20T22:15:51.314 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:51 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:51.314 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:51 smithi006 ceph-mon[24790]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-05-20T22:15:51.314 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:51 smithi006 ceph-mon[24790]: Reconfiguring daemon alertmanager.smithi006 on smithi006 2024-05-20T22:15:51.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:51 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:51.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:51 smithi078 ceph-mon[27225]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-05-20T22:15:51.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:51 smithi078 ceph-mon[27225]: Reconfiguring daemon alertmanager.smithi006 on smithi006 2024-05-20T22:15:52.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:52 smithi006 ceph-mon[24790]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:52 smithi078 ceph-mon[27225]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:54.735 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:54 smithi006 ceph-mon[24790]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:54.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:54 smithi078 ceph-mon[27225]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:55.992 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:55.992 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:55 smithi006 ceph-mon[24790]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:55.992 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:15:55.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:15:56.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:15:56.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:55 smithi078 ceph-mon[27225]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:56.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:15:56.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:15:58.226 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:57 smithi006 ceph-mon[24790]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:58.226 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:57 smithi006 ceph-mon[24790]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:15:58.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:57 smithi078 ceph-mon[27225]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:15:58.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:57 smithi078 ceph-mon[27225]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:15:59.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:58 smithi006 ceph-mon[24790]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:15:59.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:58 smithi006 ceph-mon[24790]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T22:15:59.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:58 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:15:59.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:58 smithi006 ceph-mon[24790]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:15:59.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:58 smithi078 ceph-mon[27225]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:15:59.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:58 smithi078 ceph-mon[27225]: osdmap e7: 1 total, 0 up, 1 in 2024-05-20T22:15:59.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:58 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:15:59.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:58 smithi078 ceph-mon[27225]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:16:00.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:59 smithi006 ceph-mon[24790]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:16:00.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:59 smithi006 ceph-mon[24790]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:16:00.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:59 smithi006 ceph-mon[24790]: osdmap e8: 1 total, 0 up, 1 in 2024-05-20T22:16:00.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:59 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:00.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:15:59 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:00.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:59 smithi078 ceph-mon[27225]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:16:00.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:59 smithi078 ceph-mon[27225]: from='osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:16:00.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:59 smithi078 ceph-mon[27225]: osdmap e8: 1 total, 0 up, 1 in 2024-05-20T22:16:00.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:59 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:00.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:15:59 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:01.241 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:16:01.241 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:16:01.241 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:01.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943] boot 2024-05-20T22:16:01.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T22:16:01.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:01.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:01.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: Reconfiguring crash.smithi006 (monmap changed)... 2024-05-20T22:16:01.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:16:01.243 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:01.243 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:00 smithi006 ceph-mon[24790]: Reconfiguring daemon crash.smithi006 on smithi006 2024-05-20T22:16:01.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:16:01.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:16:01.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:01.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: osd.0 [v2:172.21.15.6:6802/750456943,v1:172.21.15.6:6803/750456943] boot 2024-05-20T22:16:01.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: osdmap e9: 1 total, 1 up, 1 in 2024-05-20T22:16:01.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:16:01.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:01.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: Reconfiguring crash.smithi006 (monmap changed)... 2024-05-20T22:16:01.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:16:01.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:01.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:00 smithi078 ceph-mon[27225]: Reconfiguring daemon crash.smithi006 on smithi006 2024-05-20T22:16:01.588 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 0 on host 'smithi006' 2024-05-20T22:16:02.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:02 smithi006 ceph-mon[24790]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:16:02.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:02 smithi006 ceph-mon[24790]: osdmap e10: 1 total, 1 up, 1 in 2024-05-20T22:16:02.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:02 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:02.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:02 smithi078 ceph-mon[27225]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-05-20T22:16:02.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:02 smithi078 ceph-mon[27225]: osdmap e10: 1 total, 1 up, 1 in 2024-05-20T22:16:02.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:02 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:03.240 DEBUG:teuthology.orchestra.run.smithi006:osd.0> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.0.service 2024-05-20T22:16:03.243 INFO:tasks.cephadm:Deploying osd.1 on smithi006 with /dev/vg_nvme/lv_3... 2024-05-20T22:16:03.243 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-20T22:16:04.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:04.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:03 smithi006 ceph-mon[24790]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-05-20T22:16:04.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:03 smithi006 ceph-mon[24790]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-05-20T22:16:04.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:03 smithi006 ceph-mon[24790]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:04.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:03 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:04.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:03 smithi078 ceph-mon[27225]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-05-20T22:16:04.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:03 smithi078 ceph-mon[27225]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-05-20T22:16:04.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:03 smithi078 ceph-mon[27225]: pgmap v32: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:06.757 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:06 smithi006 ceph-mon[24790]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:06.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:06 smithi078 ceph-mon[27225]: pgmap v33: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:07.980 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-20T22:16:07.980 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-20T22:16:07.980 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2024-05-20T22:16:07.980 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2024-05-20T22:16:07.980 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111348 s, 942 MB/s 2024-05-20T22:16:07.980 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2024-05-20T22:16:08.689 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:08 smithi006 ceph-mon[24790]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:08.716 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_3 2024-05-20T22:16:08.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:08 smithi078 ceph-mon[27225]: pgmap v34: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:10.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:10 smithi006 ceph-mon[24790]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:10.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:10 smithi078 ceph-mon[27225]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:11.577 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:11 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:16:11.577 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:11 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:16:11.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:11 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:11.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:11 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:11.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:11 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:16:11.578 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:11 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:11.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:11 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:16:11.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:11 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:16:11.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:11 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:11.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:11 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:11.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:11 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:16:11.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:11 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: from='client.14260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:16:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: Reconfiguring crash.smithi078 (monmap changed)... 2024-05-20T22:16:12.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: Reconfiguring daemon crash.smithi078 on smithi078 2024-05-20T22:16:12.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:12.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:16:12.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:16:12.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:12 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:12.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:12.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: from='client.14260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:16:12.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: Reconfiguring crash.smithi078 (monmap changed)... 2024-05-20T22:16:12.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: Reconfiguring daemon crash.smithi078 on smithi078 2024-05-20T22:16:12.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:12.796 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:16:12.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:16:12.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:12 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: Reconfiguring mgr.smithi078.vxzcbt (monmap changed)... 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: Reconfiguring daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1965217769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3987af5-4f2c-4933-818b-5387358f701c"}]: dispatch 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1965217769' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b3987af5-4f2c-4933-818b-5387358f701c"}]': finished 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:16:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: Reconfiguring mgr.smithi078.vxzcbt (monmap changed)... 2024-05-20T22:16:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: Reconfiguring daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:16:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1965217769' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b3987af5-4f2c-4933-818b-5387358f701c"}]: dispatch 2024-05-20T22:16:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1965217769' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b3987af5-4f2c-4933-818b-5387358f701c"}]': finished 2024-05-20T22:16:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: osdmap e11: 2 total, 1 up, 2 in 2024-05-20T22:16:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:16:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:16:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:14.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:14 smithi078 ceph-mon[27225]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:14.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:14 smithi078 ceph-mon[27225]: Reconfiguring mon.smithi078 (monmap changed)... 2024-05-20T22:16:14.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:14 smithi078 ceph-mon[27225]: Reconfiguring daemon mon.smithi078 on smithi078 2024-05-20T22:16:14.731 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:14 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/4066849863' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:16:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:14 smithi006 ceph-mon[24790]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:14 smithi006 ceph-mon[24790]: Reconfiguring mon.smithi078 (monmap changed)... 2024-05-20T22:16:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:14 smithi006 ceph-mon[24790]: Reconfiguring daemon mon.smithi078 on smithi078 2024-05-20T22:16:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:14 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/4066849863' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:16:16.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:15 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:16.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:15 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:16:16.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:15 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:16:16.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:15 smithi006 ceph-mon[24790]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:16.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:15 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:16.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:15 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:16:16.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:15 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:16:16.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:15 smithi078 ceph-mon[27225]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:18.169 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:17 smithi006 ceph-mon[24790]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:18.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:17 smithi078 ceph-mon[27225]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:20.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:19 smithi006 ceph-mon[24790]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:20.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:19 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:16:20.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:19 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:20.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:19 smithi006 ceph-mon[24790]: Deploying daemon osd.1 on smithi006 2024-05-20T22:16:20.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:19 smithi078 ceph-mon[27225]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:20.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:19 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:16:20.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:19 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:20.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:19 smithi078 ceph-mon[27225]: Deploying daemon osd.1 on smithi006 2024-05-20T22:16:22.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:22 smithi006 ceph-mon[24790]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:22.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:22 smithi078 ceph-mon[27225]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:24.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:24 smithi006 ceph-mon[24790]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:24.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:24.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:24 smithi078 ceph-mon[27225]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:24.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:26.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:26 smithi006 ceph-mon[24790]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:26 smithi078 ceph-mon[27225]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:27.075 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 1 on host 'smithi006' 2024-05-20T22:16:27.666 DEBUG:teuthology.orchestra.run.smithi006:osd.1> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.1.service 2024-05-20T22:16:27.669 INFO:tasks.cephadm:Deploying osd.2 on smithi006 with /dev/vg_nvme/lv_2... 2024-05-20T22:16:27.669 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:16:28.292 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:28 smithi006 ceph-mon[24790]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.6:9093"}]: dispatch 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-05-20T22:16:28.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.6:3000"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.6:9095"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:16:28.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:28 smithi078 ceph-mon[27225]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:16:29.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:29 smithi078 ceph-mon[27225]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:16:29.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:29 smithi078 ceph-mon[27225]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T22:16:29.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:29.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:29 smithi078 ceph-mon[27225]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:16:29.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:29.486 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:29 smithi006 ceph-mon[24790]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:16:29.486 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:29 smithi006 ceph-mon[24790]: osdmap e12: 2 total, 1 up, 2 in 2024-05-20T22:16:29.486 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:29.486 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:29 smithi006 ceph-mon[24790]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:16:29.486 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:29.486 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:16:29 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[37994]: 2024-05-20T22:16:29.129+0000 7fabf35e7700 -1 osd.1 0 waiting for initial osdmap 2024-05-20T22:16:29.486 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:16:29 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[37994]: 2024-05-20T22:16:29.135+0000 7fabedf7e700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:16:30.197 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-20T22:16:30.197 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-20T22:16:30.197 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2024-05-20T22:16:30.197 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2024-05-20T22:16:30.197 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123092 s, 852 MB/s 2024-05-20T22:16:30.197 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2024-05-20T22:16:30.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:30 smithi078 ceph-mon[27225]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:30.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:30 smithi078 ceph-mon[27225]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:16:30.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:30 smithi078 ceph-mon[27225]: osdmap e13: 2 total, 1 up, 2 in 2024-05-20T22:16:30.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:30 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:30.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:30 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:30.432 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:30 smithi006 ceph-mon[24790]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:30.432 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:30 smithi006 ceph-mon[24790]: from='osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:16:30.433 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:30 smithi006 ceph-mon[24790]: osdmap e13: 2 total, 1 up, 2 in 2024-05-20T22:16:30.433 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:30 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:30.433 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:30 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:31.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:31 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:16:31.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:31 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:16:31.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:31.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:31 smithi006 ceph-mon[24790]: osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072] boot 2024-05-20T22:16:31.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:31 smithi006 ceph-mon[24790]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T22:16:31.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:31.296 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_2 2024-05-20T22:16:31.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:31 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:16:31.403 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:31 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:16:31.403 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:31.403 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:31 smithi078 ceph-mon[27225]: osd.1 [v2:172.21.15.6:6810/3251516072,v1:172.21.15.6:6811/3251516072] boot 2024-05-20T22:16:31.403 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:31 smithi078 ceph-mon[27225]: osdmap e14: 2 total, 2 up, 2 in 2024-05-20T22:16:31.403 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:16:32.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:32 smithi006 ceph-mon[24790]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:32.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:32 smithi006 ceph-mon[24790]: osdmap e15: 2 total, 2 up, 2 in 2024-05-20T22:16:32.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:32 smithi078 ceph-mon[27225]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-05-20T22:16:32.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:32 smithi078 ceph-mon[27225]: osdmap e15: 2 total, 2 up, 2 in 2024-05-20T22:16:34.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:34 smithi006 ceph-mon[24790]: pgmap v52: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:34.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:34 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:16:34.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:34 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:16:34.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:34 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:34.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:34 smithi078 ceph-mon[27225]: pgmap v52: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:34.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:34 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:16:34.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:34 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:16:34.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:34 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:35.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:35 smithi006 ceph-mon[24790]: from='client.14278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:16:35.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:35 smithi078 ceph-mon[27225]: from='client.14278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:16:36.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:36 smithi006 ceph-mon[24790]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:36.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:36.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:36 smithi078 ceph-mon[27225]: pgmap v53: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:36.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:37.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:37 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/96667911' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59140224-d800-4a50-81d7-711022859d52"}]: dispatch 2024-05-20T22:16:37.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:37 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/96667911' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59140224-d800-4a50-81d7-711022859d52"}]': finished 2024-05-20T22:16:37.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:37 smithi006 ceph-mon[24790]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T22:16:37.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:37 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/96667911' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59140224-d800-4a50-81d7-711022859d52"}]: dispatch 2024-05-20T22:16:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:37 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/96667911' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59140224-d800-4a50-81d7-711022859d52"}]': finished 2024-05-20T22:16:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:37 smithi078 ceph-mon[27225]: osdmap e16: 3 total, 2 up, 3 in 2024-05-20T22:16:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:38.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:38 smithi006 ceph-mon[24790]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:38.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:38 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3630319874' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:16:38.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:38 smithi078 ceph-mon[27225]: pgmap v54: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:38.552 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:38 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/3630319874' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:16:40.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:40 smithi006 ceph-mon[24790]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:40.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:40 smithi078 ceph-mon[27225]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:42.720 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:42 smithi006 ceph-mon[24790]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:42.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:42 smithi078 ceph-mon[27225]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: Detected new or changed devices on smithi006 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi006 to 6434M 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:16:43.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:43 smithi006 ceph-mon[24790]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: Detected new or changed devices on smithi006 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi006 to 6434M 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:16:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:43.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:16:43.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:43 smithi078 ceph-mon[27225]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:16:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:45 smithi078 ceph-mon[27225]: Deploying daemon osd.2 on smithi006 2024-05-20T22:16:45.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:45.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:16:45.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:45.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:45 smithi006 ceph-mon[24790]: Deploying daemon osd.2 on smithi006 2024-05-20T22:16:46.777 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:46 smithi006 ceph-mon[24790]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:46.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:46 smithi078 ceph-mon[27225]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:48.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:48 smithi006 ceph-mon[24790]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:48.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:48 smithi078 ceph-mon[27225]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:50.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:50 smithi006 ceph-mon[24790]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:50.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:50.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:50 smithi078 ceph-mon[27225]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:50.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:52.571 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:52 smithi006 ceph-mon[24790]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:52.572 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:52 smithi078 ceph-mon[27225]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:54.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:54 smithi078 ceph-mon[27225]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:54.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:54.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:54.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:16:54.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:54.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:16:54.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:54 smithi006 ceph-mon[24790]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:54.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:54.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:54.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:16:54.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:16:54.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:16:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:16:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:16:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:16:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:16:56.639 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 2 on host 'smithi006' 2024-05-20T22:16:56.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:56 smithi006 ceph-mon[24790]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:56.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:56 smithi006 ceph-mon[24790]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T22:16:56.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:56 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:56 smithi078 ceph-mon[27225]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:56 smithi078 ceph-mon[27225]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T22:16:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:56 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:57.255 DEBUG:teuthology.orchestra.run.smithi006:osd.2> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.2.service 2024-05-20T22:16:57.258 INFO:tasks.cephadm:Deploying osd.3 on smithi006 with /dev/vg_nvme/lv_1... 2024-05-20T22:16:57.258 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-20T22:16:57.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:57 smithi006 ceph-mon[24790]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T22:16:57.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:57 smithi006 ceph-mon[24790]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T22:16:57.886 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:57 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:57.886 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:57 smithi006 ceph-mon[24790]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:16:57.886 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:57 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:57.886 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:16:57 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[41821]: 2024-05-20T22:16:57.554+0000 7f0419375700 -1 osd.2 0 waiting for initial osdmap 2024-05-20T22:16:57.886 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:16:57 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[41821]: 2024-05-20T22:16:57.562+0000 7f0411d08700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:16:57.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:57 smithi078 ceph-mon[27225]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T22:16:57.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:57 smithi078 ceph-mon[27225]: osdmap e17: 3 total, 2 up, 3 in 2024-05-20T22:16:57.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:57 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:57.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:57 smithi078 ceph-mon[27225]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:16:57.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:57 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:16:58.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:58 smithi006 ceph-mon[24790]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:58.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:58 smithi006 ceph-mon[24790]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:16:58.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:58 smithi006 ceph-mon[24790]: osdmap e18: 3 total, 2 up, 3 in 2024-05-20T22:16:58.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:58 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:58.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:58 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:58.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:58 smithi078 ceph-mon[27225]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:58.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:58 smithi078 ceph-mon[27225]: from='osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:16:58.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:58 smithi078 ceph-mon[27225]: osdmap e18: 3 total, 2 up, 3 in 2024-05-20T22:16:58.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:58 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:58.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:58 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:59.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:16:59.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:16:59.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:59.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218] boot 2024-05-20T22:16:59.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T22:16:59.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:59.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:59.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:16:59 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: osd.2 [v2:172.21.15.6:6818/3612674218,v1:172.21.15.6:6819/3612674218] boot 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: osdmap e19: 3 total, 3 up, 3 in 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-05-20T22:16:59.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:16:59 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-05-20T22:17:00.286 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-20T22:17:00.286 INFO:teuthology.orchestra.run.smithi006.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-20T22:17:00.286 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10+0 records in 2024-05-20T22:17:00.286 INFO:teuthology.orchestra.run.smithi006.stderr:10+0 records out 2024-05-20T22:17:00.286 INFO:teuthology.orchestra.run.smithi006.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0119616 s, 877 MB/s 2024-05-20T22:17:00.286 INFO:teuthology.orchestra.run.smithi006.stderr:--> Zapping successful for: 2024-05-20T22:17:00.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-20T22:17:00.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:00 smithi006 ceph-mon[24790]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T22:17:00.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T22:17:00.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-05-20T22:17:00.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:00 smithi078 ceph-mon[27225]: osdmap e20: 3 total, 3 up, 3 in 2024-05-20T22:17:00.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-05-20T22:17:00.902 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi006:vg_nvme/lv_1 2024-05-20T22:17:01.540 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:17:01 smithi006 sudo[43204]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:17:01.540 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:17:01 smithi006 sudo[43204]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:01.540 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:17:01 smithi006 sudo[43204]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:01.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T22:17:01.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:01 smithi078 ceph-mon[27225]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T22:17:01.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:01 smithi078 ceph-mon[27225]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:02.040 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:17:01 smithi006 sudo[43240]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:17:02.040 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:17:01 smithi006 sudo[43240]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:02.040 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:17:01 smithi006 sudo[43240]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:02.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-05-20T22:17:02.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:01 smithi006 ceph-mon[24790]: osdmap e21: 3 total, 3 up, 3 in 2024-05-20T22:17:02.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:01 smithi006 ceph-mon[24790]: pgmap v72: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:02.411 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:17:02 smithi006 sudo[43256]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:17:02.411 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:17:02 smithi006 sudo[43256]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:02.411 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:17:02 smithi006 sudo[43256]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:02.790 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:17:02 smithi006 sudo[43287]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:17:02.790 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:17:02 smithi006 sudo[43287]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:02.790 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:17:02 smithi006 sudo[43287]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:02.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:02 smithi006 ceph-mon[24790]: osdmap e22: 3 total, 3 up, 3 in 2024-05-20T22:17:02.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:02 smithi006 ceph-mon[24790]: mgrmap e19: smithi006.ssyogp(active, since 2m), standbys: smithi078.vxzcbt 2024-05-20T22:17:02.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:02 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:02.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:02 smithi078 ceph-mon[27225]: osdmap e22: 3 total, 3 up, 3 in 2024-05-20T22:17:02.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:02 smithi078 ceph-mon[27225]: mgrmap e19: smithi006.ssyogp(active, since 2m), standbys: smithi078.vxzcbt 2024-05-20T22:17:02.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:02 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:03.190 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43345]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:17:03.190 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43345]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:03.190 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43345]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:03.790 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43421]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:17:03.790 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43421]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:03.790 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43421]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:04.148 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43477]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:17:04.148 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43477]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:04.149 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:03 smithi006 sudo[43477]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:04.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:04 smithi006 sudo[43496]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:17:04.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:04 smithi006 sudo[43496]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:04.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:04 smithi006 sudo[43496]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:04.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:04 smithi078 sudo[29505]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-05-20T22:17:04.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:04 smithi078 sudo[29505]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:04.942 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:04 smithi078 sudo[29505]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:04.956 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:04 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:05.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:04 smithi078 sudo[29508]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-05-20T22:17:05.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:04 smithi078 sudo[29508]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-05-20T22:17:05.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:04 smithi078 sudo[29508]: pam_unix(sudo:session): session closed for user root 2024-05-20T22:17:05.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:04 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: pgmap v74: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='client.14302 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:17:06.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:17:06.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:17:06.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:06.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:06.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:17:06.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: pgmap v74: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:17:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:17:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:17:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:17:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-05-20T22:17:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='client.14302 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi006:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-05-20T22:17:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:08.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi006 to 4289M 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:08.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:07 smithi006 ceph-mon[24790]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 5.1 KiB/s rd, 245 KiB/s wr, 16 op/s 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi006 to 4289M 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:08.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:08.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:08.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:07 smithi078 ceph-mon[27225]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 5.1 KiB/s rd, 245 KiB/s wr, 16 op/s 2024-05-20T22:17:08.992 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:08 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1068916510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7f4845a-1617-47ab-be52-1efc2c8e3bca"}]: dispatch 2024-05-20T22:17:08.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:08 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1068916510' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7f4845a-1617-47ab-be52-1efc2c8e3bca"}]': finished 2024-05-20T22:17:08.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:08 smithi006 ceph-mon[24790]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T22:17:08.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:09.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:08 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1068916510' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d7f4845a-1617-47ab-be52-1efc2c8e3bca"}]: dispatch 2024-05-20T22:17:09.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:08 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1068916510' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d7f4845a-1617-47ab-be52-1efc2c8e3bca"}]': finished 2024-05-20T22:17:09.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:08 smithi078 ceph-mon[27225]: osdmap e23: 4 total, 3 up, 4 in 2024-05-20T22:17:09.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:08 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:09.763 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:09 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/4111332531' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:17:09.763 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:09 smithi006 ceph-mon[24790]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 178 KiB/s wr, 12 op/s 2024-05-20T22:17:10.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:09 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/4111332531' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:17:10.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:09 smithi078 ceph-mon[27225]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 178 KiB/s wr, 12 op/s 2024-05-20T22:17:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:12 smithi006 ceph-mon[24790]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 140 KiB/s wr, 9 op/s 2024-05-20T22:17:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:12 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:12 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:12.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:12 smithi078 ceph-mon[27225]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 140 KiB/s wr, 9 op/s 2024-05-20T22:17:12.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:12 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:12.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:12 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:14.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:14 smithi006 ceph-mon[24790]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-20T22:17:14.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:14 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:17:14.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:14 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:14.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:14 smithi078 ceph-mon[27225]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-20T22:17:14.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:14 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:17:14.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:14 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:15.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:15 smithi078 ceph-mon[27225]: Deploying daemon osd.3 on smithi006 2024-05-20T22:17:15.902 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:15 smithi006 ceph-mon[24790]: Deploying daemon osd.3 on smithi006 2024-05-20T22:17:16.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:16 smithi006 ceph-mon[24790]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-20T22:17:16.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:16 smithi078 ceph-mon[27225]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-05-20T22:17:18.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:18 smithi006 ceph-mon[24790]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:18.812 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:18 smithi078 ceph-mon[27225]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:19.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:19 smithi078 ceph-mon[27225]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:20.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:19 smithi006 ceph-mon[24790]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:21.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:20 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:21.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:20 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:21.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:20 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:21.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:20 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:21.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:20 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:21.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:20 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:21.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:20 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:21.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:20 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:22.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:21 smithi006 ceph-mon[24790]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:22.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:21 smithi078 ceph-mon[27225]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:22.796 INFO:teuthology.orchestra.run.smithi006.stdout:Created osd(s) 3 on host 'smithi006' 2024-05-20T22:17:23.571 DEBUG:teuthology.orchestra.run.smithi006:osd.3> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.3.service 2024-05-20T22:17:23.574 INFO:tasks.cephadm:Deploying osd.4 on smithi078 with /dev/vg_nvme/lv_4... 2024-05-20T22:17:23.574 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-05-20T22:17:23.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:23 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:23.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:23 smithi078 ceph-mon[27225]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:17:23.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:23 smithi078 ceph-mon[27225]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:24.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:23 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:24.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:23 smithi006 ceph-mon[24790]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:17:24.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:23 smithi006 ceph-mon[24790]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:24.939 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:24 smithi006 ceph-mon[24790]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T22:17:24.939 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:24 smithi006 ceph-mon[24790]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T22:17:24.939 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:24.939 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:24 smithi006 ceph-mon[24790]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:17:24.940 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:17:24 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[45688]: 2024-05-20T22:17:24.805+0000 7f7faf212700 -1 osd.3 0 waiting for initial osdmap 2024-05-20T22:17:24.940 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:17:24 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[45688]: 2024-05-20T22:17:24.813+0000 7f7fa9ba9700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:17:25.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:24 smithi078 ceph-mon[27225]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T22:17:25.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:24 smithi078 ceph-mon[27225]: osdmap e24: 4 total, 3 up, 4 in 2024-05-20T22:17:25.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:25.045 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:24 smithi078 ceph-mon[27225]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:17:25.507 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-05-20T22:17:25.507 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-05-20T22:17:25.508 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2024-05-20T22:17:25.508 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2024-05-20T22:17:25.508 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114317 s, 917 MB/s 2024-05-20T22:17:25.508 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2024-05-20T22:17:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:25 smithi006 ceph-mon[24790]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:17:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:25 smithi006 ceph-mon[24790]: osdmap e25: 4 total, 3 up, 4 in 2024-05-20T22:17:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:26.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:25 smithi006 ceph-mon[24790]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:26.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:25 smithi078 ceph-mon[27225]: from='osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]': finished 2024-05-20T22:17:26.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:25 smithi078 ceph-mon[27225]: osdmap e25: 4 total, 3 up, 4 in 2024-05-20T22:17:26.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:26.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:26.093 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:25 smithi078 ceph-mon[27225]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-05-20T22:17:26.120 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_4 2024-05-20T22:17:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:26 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:17:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:26 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:17:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:26 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:26 smithi078 ceph-mon[27225]: osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948] boot 2024-05-20T22:17:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:26 smithi078 ceph-mon[27225]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T22:17:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:26 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:26 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:27.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:26 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:17:27.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:26 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:17:27.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:26 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:27.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:26 smithi006 ceph-mon[24790]: osd.3 [v2:172.21.15.6:6826/2017175948,v1:172.21.15.6:6827/2017175948] boot 2024-05-20T22:17:27.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:26 smithi006 ceph-mon[24790]: osdmap e26: 4 total, 4 up, 4 in 2024-05-20T22:17:27.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:26 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:17:27.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:26 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:27.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:27 smithi006 ceph-mon[24790]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T22:17:27.951 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:27 smithi006 ceph-mon[24790]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:28.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:27 smithi078 ceph-mon[27225]: osdmap e27: 4 total, 4 up, 4 in 2024-05-20T22:17:28.018 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:27 smithi078 ceph-mon[27225]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:28.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:28 smithi078 ceph-mon[27225]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:17:28.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:17:28.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:17:28.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:28 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:29.032 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:28 smithi006 ceph-mon[24790]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:17:29.032 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:17:29.032 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:17:29.032 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:28 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:29.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:29.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:30.223 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:30.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:30.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:30.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:31.132 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:30 smithi006 ceph-mon[24790]: Detected new or changed devices on smithi006 2024-05-20T22:17:31.132 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:30 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi006 to 3217M 2024-05-20T22:17:31.132 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:30 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/3719580336' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "906346ce-6f48-4cc2-8cd7-f8c03e431dcb"}]: dispatch 2024-05-20T22:17:31.132 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:30 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "906346ce-6f48-4cc2-8cd7-f8c03e431dcb"}]: dispatch 2024-05-20T22:17:31.132 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:30 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "906346ce-6f48-4cc2-8cd7-f8c03e431dcb"}]': finished 2024-05-20T22:17:31.132 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:30 smithi006 ceph-mon[24790]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T22:17:31.133 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:30 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:31.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:30 smithi078 ceph-mon[27225]: Detected new or changed devices on smithi006 2024-05-20T22:17:31.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:30 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi006 to 3217M 2024-05-20T22:17:31.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:30 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/3719580336' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "906346ce-6f48-4cc2-8cd7-f8c03e431dcb"}]: dispatch 2024-05-20T22:17:31.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:30 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "906346ce-6f48-4cc2-8cd7-f8c03e431dcb"}]: dispatch 2024-05-20T22:17:31.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:30 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "906346ce-6f48-4cc2-8cd7-f8c03e431dcb"}]': finished 2024-05-20T22:17:31.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:30 smithi078 ceph-mon[27225]: osdmap e28: 5 total, 4 up, 5 in 2024-05-20T22:17:31.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:30 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:32.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:31 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/3141541666' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:17:32.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:31 smithi006 ceph-mon[24790]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:32.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:31 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/3141541666' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:17:32.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:31 smithi078 ceph-mon[27225]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:34.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:33 smithi006 ceph-mon[24790]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:34.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:33 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:34.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:33 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:34.262 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:33 smithi078 ceph-mon[27225]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:33 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:34.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:33 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:36.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:36 smithi078 ceph-mon[27225]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:36.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:17:36.797 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:37.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:36 smithi006 ceph-mon[24790]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:37.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:17:37.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:37.822 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:37 smithi078 ceph-mon[27225]: Deploying daemon osd.4 on smithi078 2024-05-20T22:17:38.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:37 smithi006 ceph-mon[24790]: Deploying daemon osd.4 on smithi078 2024-05-20T22:17:38.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:38 smithi078 ceph-mon[27225]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:39.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:38 smithi006 ceph-mon[24790]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:40.836 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:40 smithi078 ceph-mon[27225]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:41.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:40 smithi006 ceph-mon[24790]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:42.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:42 smithi006 ceph-mon[24790]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:42.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:42.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:42.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:42.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:42 smithi078 ceph-mon[27225]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:44.482 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:44 smithi078 ceph-mon[27225]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:44.514 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 4 on host 'smithi078' 2024-05-20T22:17:44.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:44 smithi006 ceph-mon[24790]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:45.059 DEBUG:teuthology.orchestra.run.smithi078:osd.4> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.4.service 2024-05-20T22:17:45.062 INFO:tasks.cephadm:Deploying osd.5 on smithi078 with /dev/vg_nvme/lv_3... 2024-05-20T22:17:45.062 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-05-20T22:17:45.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:45 smithi078 ceph-mon[27225]: from='osd.4 [v2:172.21.15.78:6800/1107461992,v1:172.21.15.78:6801/1107461992]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:17:45.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:45 smithi078 ceph-mon[27225]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:17:45.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:45 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:45.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:45 smithi006 ceph-mon[24790]: from='osd.4 [v2:172.21.15.78:6800/1107461992,v1:172.21.15.78:6801/1107461992]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:17:45.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:45 smithi006 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:17:45.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:45 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:46.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:46 smithi078 ceph-mon[27225]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:46.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:46 smithi078 ceph-mon[27225]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T22:17:46.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:46 smithi078 ceph-mon[27225]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T22:17:46.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:46 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:46.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:46 smithi078 ceph-mon[27225]: from='osd.4 [v2:172.21.15.78:6800/1107461992,v1:172.21.15.78:6801/1107461992]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:17:46.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:46 smithi078 ceph-mon[27225]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:17:46.539 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:46 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:46.540 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:17:46 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[31426]: 2024-05-20T22:17:46.171+0000 7f6a16107700 -1 osd.4 0 waiting for initial osdmap 2024-05-20T22:17:46.540 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:17:46 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[31426]: 2024-05-20T22:17:46.175+0000 7f6a10a9e700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:17:46.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:46 smithi006 ceph-mon[24790]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:46.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:46 smithi006 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T22:17:46.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:46 smithi006 ceph-mon[24790]: osdmap e29: 5 total, 4 up, 5 in 2024-05-20T22:17:46.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:46 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:46.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:46 smithi006 ceph-mon[24790]: from='osd.4 [v2:172.21.15.78:6800/1107461992,v1:172.21.15.78:6801/1107461992]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:17:46.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:46 smithi006 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:17:46.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:46 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:47.197 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-05-20T22:17:47.197 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-05-20T22:17:47.198 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2024-05-20T22:17:47.198 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2024-05-20T22:17:47.198 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012988 s, 807 MB/s 2024-05-20T22:17:47.198 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2024-05-20T22:17:47.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:47 smithi078 ceph-mon[27225]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:17:47.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:47 smithi078 ceph-mon[27225]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T22:17:47.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:47.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:47.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:47 smithi006 ceph-mon[24790]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:17:47.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:47 smithi006 ceph-mon[24790]: osdmap e30: 5 total, 4 up, 5 in 2024-05-20T22:17:47.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:47.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:48.085 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_3 2024-05-20T22:17:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:48 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:17:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:48 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:17:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:48 smithi078 ceph-mon[27225]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:48 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:48 smithi078 ceph-mon[27225]: osd.4 [v2:172.21.15.78:6800/1107461992,v1:172.21.15.78:6801/1107461992] boot 2024-05-20T22:17:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:48 smithi078 ceph-mon[27225]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T22:17:48.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:48 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:48.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:48 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:17:48.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:48 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:17:48.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:48 smithi006 ceph-mon[24790]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-05-20T22:17:48.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:48 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:48.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:48 smithi006 ceph-mon[24790]: osd.4 [v2:172.21.15.78:6800/1107461992,v1:172.21.15.78:6801/1107461992] boot 2024-05-20T22:17:48.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:48 smithi006 ceph-mon[24790]: osdmap e31: 5 total, 5 up, 5 in 2024-05-20T22:17:48.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:48 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:17:49.390 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:49 smithi078 ceph-mon[27225]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T22:17:49.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:49 smithi006 ceph-mon[24790]: osdmap e32: 5 total, 5 up, 5 in 2024-05-20T22:17:50.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: osdmap e33: 5 total, 5 up, 5 in 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:50.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: osdmap e33: 5 total, 5 up, 5 in 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:17:50.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:17:51.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:51 smithi078 ceph-mon[27225]: Detected new or changed devices on smithi078 2024-05-20T22:17:51.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:51 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi078 to 15936M 2024-05-20T22:17:51.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:51 smithi078 ceph-mon[27225]: osdmap e34: 5 total, 5 up, 5 in 2024-05-20T22:17:51.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:51 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:17:51.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:51 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:17:51.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:51 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:51.507 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:51 smithi078 ceph-mon[27225]: osdmap e35: 5 total, 5 up, 5 in 2024-05-20T22:17:51.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:51 smithi006 ceph-mon[24790]: Detected new or changed devices on smithi078 2024-05-20T22:17:51.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:51 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi078 to 15936M 2024-05-20T22:17:51.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:51 smithi006 ceph-mon[24790]: osdmap e34: 5 total, 5 up, 5 in 2024-05-20T22:17:51.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:51 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:17:51.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:51 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:17:51.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:51 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:17:51.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:51 smithi006 ceph-mon[24790]: osdmap e35: 5 total, 5 up, 5 in 2024-05-20T22:17:52.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:52 smithi078 ceph-mon[27225]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:17:52.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:52 smithi078 ceph-mon[27225]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:17:52.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:52 smithi006 ceph-mon[24790]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:17:52.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:52 smithi006 ceph-mon[24790]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:17:53.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:53 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/2439787213' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02c0433e-9f55-4164-8316-92a948cd0e8f"}]: dispatch 2024-05-20T22:17:53.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:53 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02c0433e-9f55-4164-8316-92a948cd0e8f"}]: dispatch 2024-05-20T22:17:53.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:53 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02c0433e-9f55-4164-8316-92a948cd0e8f"}]': finished 2024-05-20T22:17:53.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:53 smithi006 ceph-mon[24790]: osdmap e36: 6 total, 5 up, 6 in 2024-05-20T22:17:53.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:53 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:17:53.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:53 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/2439787213' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02c0433e-9f55-4164-8316-92a948cd0e8f"}]: dispatch 2024-05-20T22:17:53.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:53 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "02c0433e-9f55-4164-8316-92a948cd0e8f"}]: dispatch 2024-05-20T22:17:53.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:53 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "02c0433e-9f55-4164-8316-92a948cd0e8f"}]': finished 2024-05-20T22:17:53.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:53 smithi078 ceph-mon[27225]: osdmap e36: 6 total, 5 up, 6 in 2024-05-20T22:17:53.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:53 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:17:54.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:54 smithi078 ceph-mon[27225]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-05-20T22:17:54.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:54 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/83468505' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:17:55.043 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:54 smithi006 ceph-mon[24790]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-05-20T22:17:55.043 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:54 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/83468505' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:17:55.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:17:55.903 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:17:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:17:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:17:56.831 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:56 smithi078 ceph-mon[27225]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-20T22:17:57.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:56 smithi006 ceph-mon[24790]: pgmap v114: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-05-20T22:17:58.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:58 smithi078 ceph-mon[27225]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-05-20T22:17:59.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:58 smithi006 ceph-mon[24790]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 77 KiB/s, 0 objects/s recovering 2024-05-20T22:17:59.868 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:17:59 smithi078 ceph-mon[27225]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-20T22:18:00.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:17:59 smithi006 ceph-mon[24790]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-20T22:18:00.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:18:00.829 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:00 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:00.830 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:00 smithi078 ceph-mon[27225]: Deploying daemon osd.5 on smithi078 2024-05-20T22:18:01.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:18:01.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:00 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:01.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:00 smithi006 ceph-mon[24790]: Deploying daemon osd.5 on smithi078 2024-05-20T22:18:01.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:01 smithi078 ceph-mon[27225]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:02.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:01 smithi006 ceph-mon[24790]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:04.090 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:03 smithi078 ceph-mon[27225]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:04.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:03 smithi006 ceph-mon[24790]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:05.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:05.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:05.957 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:05.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:05 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:05.958 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:05 smithi078 ceph-mon[27225]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:06.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:06.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:06.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:06.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:05 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:06.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:05 smithi006 ceph-mon[24790]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:08.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:07 smithi078 ceph-mon[27225]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:08.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:07 smithi078 ceph-mon[27225]: from='osd.5 [v2:172.21.15.78:6808/476389303,v1:172.21.15.78:6809/476389303]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:18:08.263 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:07 smithi078 ceph-mon[27225]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:18:08.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:07 smithi006 ceph-mon[24790]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:08.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:07 smithi006 ceph-mon[24790]: from='osd.5 [v2:172.21.15.78:6808/476389303,v1:172.21.15.78:6809/476389303]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:18:08.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:07 smithi006 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:18:08.301 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 5 on host 'smithi078' 2024-05-20T22:18:08.986 DEBUG:teuthology.orchestra.run.smithi078:osd.5> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.5.service 2024-05-20T22:18:08.988 INFO:tasks.cephadm:Deploying osd.6 on smithi078 with /dev/vg_nvme/lv_2... 2024-05-20T22:18:08.989 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-05-20T22:18:09.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:08 smithi078 ceph-mon[27225]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T22:18:09.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:08 smithi078 ceph-mon[27225]: osdmap e37: 6 total, 5 up, 6 in 2024-05-20T22:18:09.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:08 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:09.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:08 smithi078 ceph-mon[27225]: from='osd.5 [v2:172.21.15.78:6808/476389303,v1:172.21.15.78:6809/476389303]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:09.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:08 smithi078 ceph-mon[27225]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:09.235 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:08 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:09.236 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:08 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:09.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:08 smithi006 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T22:18:09.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:08 smithi006 ceph-mon[24790]: osdmap e37: 6 total, 5 up, 6 in 2024-05-20T22:18:09.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:08 smithi006 ceph-mon[24790]: from='osd.5 [v2:172.21.15.78:6808/476389303,v1:172.21.15.78:6809/476389303]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:08 smithi006 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:09.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:08 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:10.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:09 smithi078 ceph-mon[27225]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:10.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:09 smithi078 ceph-mon[27225]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:18:10.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:09 smithi078 ceph-mon[27225]: osdmap e38: 6 total, 5 up, 6 in 2024-05-20T22:18:10.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:09 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:10.132 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:09 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:10.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:09 smithi006 ceph-mon[24790]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-05-20T22:18:10.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:09 smithi006 ceph-mon[24790]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:18:10.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:09 smithi006 ceph-mon[24790]: osdmap e38: 6 total, 5 up, 6 in 2024-05-20T22:18:10.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:09 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:10.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:09 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:11.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:10 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:18:11.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:10 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:18:11.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:10 smithi006 ceph-mon[24790]: osd.5 [v2:172.21.15.78:6808/476389303,v1:172.21.15.78:6809/476389303] boot 2024-05-20T22:18:11.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:10 smithi006 ceph-mon[24790]: osdmap e39: 6 total, 6 up, 6 in 2024-05-20T22:18:11.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:10 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:11.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:10 smithi006 ceph-mon[24790]: osdmap e40: 6 total, 6 up, 6 in 2024-05-20T22:18:11.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:10 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:18:11.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:10 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:18:11.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:10 smithi078 ceph-mon[27225]: osd.5 [v2:172.21.15.78:6808/476389303,v1:172.21.15.78:6809/476389303] boot 2024-05-20T22:18:11.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:10 smithi078 ceph-mon[27225]: osdmap e39: 6 total, 6 up, 6 in 2024-05-20T22:18:11.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:10 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:18:11.313 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:10 smithi078 ceph-mon[27225]: osdmap e40: 6 total, 6 up, 6 in 2024-05-20T22:18:12.649 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-05-20T22:18:12.649 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-05-20T22:18:12.649 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2024-05-20T22:18:12.649 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2024-05-20T22:18:12.649 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0139731 s, 750 MB/s 2024-05-20T22:18:12.649 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2024-05-20T22:18:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:12 smithi006 ceph-mon[24790]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:12.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:12 smithi006 ceph-mon[24790]: osdmap e41: 6 total, 6 up, 6 in 2024-05-20T22:18:12.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:12 smithi078 ceph-mon[27225]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:12.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:12 smithi078 ceph-mon[27225]: osdmap e41: 6 total, 6 up, 6 in 2024-05-20T22:18:13.416 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_2 2024-05-20T22:18:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: osdmap e42: 6 total, 6 up, 6 in 2024-05-20T22:18:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:13.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:13.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:13.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:13.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:13.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:13.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:13.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:13 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: osdmap e42: 6 total, 6 up, 6 in 2024-05-20T22:18:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:13.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:13 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:14.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:14 smithi078 ceph-mon[27225]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:14.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:14 smithi078 ceph-mon[27225]: Detected new or changed devices on smithi078 2024-05-20T22:18:14.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:14 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi078 to 7968M 2024-05-20T22:18:14.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:14 smithi078 ceph-mon[27225]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-20T22:18:14.708 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:14 smithi078 ceph-mon[27225]: osdmap e43: 6 total, 6 up, 6 in 2024-05-20T22:18:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:14 smithi006 ceph-mon[24790]: pgmap v129: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:14 smithi006 ceph-mon[24790]: Detected new or changed devices on smithi078 2024-05-20T22:18:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:14 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi078 to 7968M 2024-05-20T22:18:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:14 smithi006 ceph-mon[24790]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-20T22:18:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:14 smithi006 ceph-mon[24790]: osdmap e43: 6 total, 6 up, 6 in 2024-05-20T22:18:15.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:15 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:18:15.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:15 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:18:15.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:15 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:16.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:15 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:18:16.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:15 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:18:16.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:15 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:16.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:16 smithi078 ceph-mon[27225]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:16.889 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:16 smithi078 ceph-mon[27225]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:18:17.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:16 smithi006 ceph-mon[24790]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:17.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:16 smithi006 ceph-mon[24790]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:18:17.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:17 smithi078 ceph-mon[27225]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:18.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:17 smithi006 ceph-mon[24790]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-20T22:18:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: Cluster is now healthy 2024-05-20T22:18:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/4090701871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4633543-b502-44cd-af23-96bdefbd1688"}]: dispatch 2024-05-20T22:18:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4633543-b502-44cd-af23-96bdefbd1688"}]: dispatch 2024-05-20T22:18:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4633543-b502-44cd-af23-96bdefbd1688"}]': finished 2024-05-20T22:18:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: osdmap e44: 7 total, 6 up, 7 in 2024-05-20T22:18:18.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:18.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:18 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/2049952875' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:18:19.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-20T22:18:19.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: Cluster is now healthy 2024-05-20T22:18:19.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/4090701871' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4633543-b502-44cd-af23-96bdefbd1688"}]: dispatch 2024-05-20T22:18:19.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4633543-b502-44cd-af23-96bdefbd1688"}]: dispatch 2024-05-20T22:18:19.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4633543-b502-44cd-af23-96bdefbd1688"}]': finished 2024-05-20T22:18:19.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: osdmap e44: 7 total, 6 up, 7 in 2024-05-20T22:18:19.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:19.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:18 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/2049952875' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:18:19.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:19 smithi078 ceph-mon[27225]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-05-20T22:18:20.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:19 smithi006 ceph-mon[24790]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-05-20T22:18:22.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:22 smithi078 ceph-mon[27225]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-20T22:18:22.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:22 smithi006 ceph-mon[24790]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-20T22:18:24.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:24 smithi078 ceph-mon[27225]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-05-20T22:18:24.878 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:18:24.878 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:25.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:24 smithi006 ceph-mon[24790]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-05-20T22:18:25.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:18:25.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:25.818 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:25 smithi078 ceph-mon[27225]: Deploying daemon osd.6 on smithi078 2024-05-20T22:18:26.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:25 smithi006 ceph-mon[24790]: Deploying daemon osd.6 on smithi078 2024-05-20T22:18:26.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:26 smithi078 ceph-mon[27225]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-20T22:18:27.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:26 smithi006 ceph-mon[24790]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-20T22:18:28.853 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:28 smithi078 ceph-mon[27225]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:29.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:28 smithi006 ceph-mon[24790]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:30.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:30.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:30.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:30.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:29 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:30.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:29 smithi078 ceph-mon[27225]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:30.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:30.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:30.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:30.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:29 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:30.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:29 smithi006 ceph-mon[24790]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:32.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:32 smithi078 ceph-mon[27225]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:32.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:32 smithi078 ceph-mon[27225]: from='osd.6 [v2:172.21.15.78:6816/2649823748,v1:172.21.15.78:6817/2649823748]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:18:32.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:32 smithi078 ceph-mon[27225]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:18:32.699 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 6 on host 'smithi078' 2024-05-20T22:18:33.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:32 smithi006 ceph-mon[24790]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:33.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:32 smithi006 ceph-mon[24790]: from='osd.6 [v2:172.21.15.78:6816/2649823748,v1:172.21.15.78:6817/2649823748]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:18:33.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:32 smithi006 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:18:33.618 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:33 smithi078 ceph-mon[27225]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T22:18:33.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:33 smithi078 ceph-mon[27225]: osdmap e45: 7 total, 6 up, 7 in 2024-05-20T22:18:33.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:33 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:33.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:33 smithi078 ceph-mon[27225]: from='osd.6 [v2:172.21.15.78:6816/2649823748,v1:172.21.15.78:6817/2649823748]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:33.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:33 smithi078 ceph-mon[27225]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:33.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:33 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:33.642 DEBUG:teuthology.orchestra.run.smithi078:osd.6> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.6.service 2024-05-20T22:18:33.644 INFO:tasks.cephadm:Deploying osd.7 on smithi078 with /dev/vg_nvme/lv_1... 2024-05-20T22:18:33.645 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-05-20T22:18:33.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:33 smithi006 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T22:18:33.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:33 smithi006 ceph-mon[24790]: osdmap e45: 7 total, 6 up, 7 in 2024-05-20T22:18:33.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:33 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:33.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:33 smithi006 ceph-mon[24790]: from='osd.6 [v2:172.21.15.78:6816/2649823748,v1:172.21.15.78:6817/2649823748]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:33.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:33 smithi006 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:33.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:33 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:18:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: osdmap e46: 7 total, 6 up, 7 in 2024-05-20T22:18:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:34.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: osd.6 [v2:172.21.15.78:6816/2649823748,v1:172.21.15.78:6817/2649823748] boot 2024-05-20T22:18:34.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: osdmap e47: 7 total, 7 up, 7 in 2024-05-20T22:18:34.902 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:34 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:35.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-05-20T22:18:35.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:18:35.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: osdmap e46: 7 total, 6 up, 7 in 2024-05-20T22:18:35.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:35.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:35.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:35.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: osd.6 [v2:172.21.15.78:6816/2649823748,v1:172.21.15.78:6817/2649823748] boot 2024-05-20T22:18:35.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: osdmap e47: 7 total, 7 up, 7 in 2024-05-20T22:18:35.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:34 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:18:35.772 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-05-20T22:18:35.772 INFO:teuthology.orchestra.run.smithi078.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-05-20T22:18:35.772 INFO:teuthology.orchestra.run.smithi078.stderr: stderr: 10+0 records in 2024-05-20T22:18:35.772 INFO:teuthology.orchestra.run.smithi078.stderr:10+0 records out 2024-05-20T22:18:35.772 INFO:teuthology.orchestra.run.smithi078.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0153827 s, 682 MB/s 2024-05-20T22:18:35.772 INFO:teuthology.orchestra.run.smithi078.stderr:--> Zapping successful for: 2024-05-20T22:18:35.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:35 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:18:35.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:35 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:18:35.865 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:35 smithi078 ceph-mon[27225]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:35.866 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:35 smithi078 ceph-mon[27225]: osdmap e48: 7 total, 7 up, 7 in 2024-05-20T22:18:36.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:35 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:18:36.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:35 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:18:36.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:35 smithi006 ceph-mon[24790]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:36.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:35 smithi006 ceph-mon[24790]: osdmap e48: 7 total, 7 up, 7 in 2024-05-20T22:18:36.856 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch daemon add osd smithi078:vg_nvme/lv_1 2024-05-20T22:18:37.802 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:37 smithi078 ceph-mon[27225]: osdmap e49: 7 total, 7 up, 7 in 2024-05-20T22:18:38.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:37 smithi006 ceph-mon[24790]: osdmap e49: 7 total, 7 up, 7 in 2024-05-20T22:18:38.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:38 smithi078 ceph-mon[27225]: pgmap v148: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:38.877 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:38 smithi078 ceph-mon[27225]: osdmap e50: 7 total, 7 up, 7 in 2024-05-20T22:18:39.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:38 smithi006 ceph-mon[24790]: pgmap v148: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:39.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:38 smithi006 ceph-mon[24790]: osdmap e50: 7 total, 7 up, 7 in 2024-05-20T22:18:39.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:39 smithi078 ceph-mon[27225]: osdmap e51: 7 total, 7 up, 7 in 2024-05-20T22:18:39.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:39 smithi078 ceph-mon[27225]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:39.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:39 smithi078 ceph-mon[27225]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:18:39.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:39 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:18:39.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:39 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:18:39.874 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:39 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:40.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:39 smithi006 ceph-mon[24790]: osdmap e51: 7 total, 7 up, 7 in 2024-05-20T22:18:40.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:39 smithi006 ceph-mon[24790]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:40.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:39 smithi006 ceph-mon[24790]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi078:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:18:40.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:39 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-05-20T22:18:40.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:39 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-05-20T22:18:40.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:39 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:41.030 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: Detected new or changed devices on smithi078 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi078 to 5312M 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:41.031 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:41.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: Detected new or changed devices on smithi078 2024-05-20T22:18:41.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi078 to 5312M 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:41.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:42.192 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:41 smithi006 ceph-mon[24790]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T22:18:42.750 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:41 smithi078 ceph-mon[27225]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-05-20T22:18:43.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:42 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/1980437466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f06cc0b-609c-4b5f-945c-89cffe7b063a"}]: dispatch 2024-05-20T22:18:43.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:42 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f06cc0b-609c-4b5f-945c-89cffe7b063a"}]: dispatch 2024-05-20T22:18:43.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:42 smithi078 ceph-mon[27225]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f06cc0b-609c-4b5f-945c-89cffe7b063a"}]': finished 2024-05-20T22:18:43.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:42 smithi078 ceph-mon[27225]: osdmap e52: 8 total, 7 up, 8 in 2024-05-20T22:18:43.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:42 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:43.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:42 smithi078 ceph-mon[27225]: from='client.? 172.21.15.78:0/704251326' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:18:43.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:42 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/1980437466' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f06cc0b-609c-4b5f-945c-89cffe7b063a"}]: dispatch 2024-05-20T22:18:43.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:42 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4f06cc0b-609c-4b5f-945c-89cffe7b063a"}]: dispatch 2024-05-20T22:18:43.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:42 smithi006 ceph-mon[24790]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4f06cc0b-609c-4b5f-945c-89cffe7b063a"}]': finished 2024-05-20T22:18:43.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:42 smithi006 ceph-mon[24790]: osdmap e52: 8 total, 7 up, 8 in 2024-05-20T22:18:43.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:42 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:43.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:42 smithi006 ceph-mon[24790]: from='client.? 172.21.15.78:0/704251326' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-05-20T22:18:44.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:43 smithi006 ceph-mon[24790]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-20T22:18:44.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:43 smithi078 ceph-mon[27225]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-20T22:18:46.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:46 smithi078 ceph-mon[27225]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T22:18:47.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:46 smithi006 ceph-mon[24790]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-05-20T22:18:48.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:48 smithi078 ceph-mon[27225]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-05-20T22:18:48.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:48 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:18:48.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:48 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:49.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:48 smithi006 ceph-mon[24790]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-05-20T22:18:49.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:48 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:18:49.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:48 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:49.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:49 smithi078 ceph-mon[27225]: Deploying daemon osd.7 on smithi078 2024-05-20T22:18:50.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:49 smithi006 ceph-mon[24790]: Deploying daemon osd.7 on smithi078 2024-05-20T22:18:50.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:50 smithi078 ceph-mon[27225]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-20T22:18:51.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:50 smithi006 ceph-mon[24790]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-05-20T22:18:52.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:52 smithi078 ceph-mon[27225]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:53.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:52 smithi006 ceph-mon[24790]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:53.780 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:53 smithi078 ceph-mon[27225]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:53.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:53 smithi006 ceph-mon[24790]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:55.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:55.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:55.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:55.015 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:54 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:55.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:55.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:18:55.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:18:55.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:54 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:18:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:55 smithi006 ceph-mon[24790]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:18:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:18:56.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:55 smithi078 ceph-mon[27225]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:56.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:18:56.047 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:18:57.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:56 smithi078 ceph-mon[27225]: from='osd.7 [v2:172.21.15.78:6824/3693180321,v1:172.21.15.78:6825/3693180321]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:18:57.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:56 smithi078 ceph-mon[27225]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:18:57.226 INFO:teuthology.orchestra.run.smithi078.stdout:Created osd(s) 7 on host 'smithi078' 2024-05-20T22:18:57.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:56 smithi006 ceph-mon[24790]: from='osd.7 [v2:172.21.15.78:6824/3693180321,v1:172.21.15.78:6825/3693180321]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:18:57.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:56 smithi006 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:18:57.907 DEBUG:teuthology.orchestra.run.smithi078:osd.7> sudo journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.7.service 2024-05-20T22:18:57.909 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-05-20T22:18:57.910 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd stat -f json 2024-05-20T22:18:58.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:57 smithi078 ceph-mon[27225]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T22:18:58.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:57 smithi078 ceph-mon[27225]: osdmap e53: 8 total, 7 up, 8 in 2024-05-20T22:18:58.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:57 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:58.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:57 smithi078 ceph-mon[27225]: from='osd.7 [v2:172.21.15.78:6824/3693180321,v1:172.21.15.78:6825/3693180321]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:58.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:57 smithi078 ceph-mon[27225]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:58.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:57 smithi078 ceph-mon[27225]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:58.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:57 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:58.237 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:57 smithi006 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T22:18:58.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:57 smithi006 ceph-mon[24790]: osdmap e53: 8 total, 7 up, 8 in 2024-05-20T22:18:58.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:57 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:58.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:57 smithi006 ceph-mon[24790]: from='osd.7 [v2:172.21.15.78:6824/3693180321,v1:172.21.15.78:6825/3693180321]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:58.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:57 smithi006 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:18:58.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:57 smithi006 ceph-mon[24790]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:18:58.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:57 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:59.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:58 smithi078 ceph-mon[27225]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:18:59.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:58 smithi078 ceph-mon[27225]: osdmap e54: 8 total, 7 up, 8 in 2024-05-20T22:18:59.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:58 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:59.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:58 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:59.074 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:58 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:59.271 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:58 smithi006 ceph-mon[24790]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]': finished 2024-05-20T22:18:59.271 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:58 smithi006 ceph-mon[24790]: osdmap e54: 8 total, 7 up, 8 in 2024-05-20T22:18:59.271 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:58 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:59.271 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:58 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:18:59.271 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:58 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:18:59.993 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:00.051 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:59 smithi078 ceph-mon[27225]: purged_snaps scrub starts 2024-05-20T22:19:00.051 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:59 smithi078 ceph-mon[27225]: purged_snaps scrub ok 2024-05-20T22:19:00.052 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:59 smithi078 ceph-mon[27225]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:19:00.052 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:59 smithi078 ceph-mon[27225]: osd.7 [v2:172.21.15.78:6824/3693180321,v1:172.21.15.78:6825/3693180321] boot 2024-05-20T22:19:00.052 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:59 smithi078 ceph-mon[27225]: osdmap e55: 8 total, 8 up, 8 in 2024-05-20T22:19:00.052 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:18:59 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:19:00.262 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:59 smithi006 ceph-mon[24790]: purged_snaps scrub starts 2024-05-20T22:19:00.262 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:59 smithi006 ceph-mon[24790]: purged_snaps scrub ok 2024-05-20T22:19:00.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:59 smithi006 ceph-mon[24790]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-05-20T22:19:00.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:59 smithi006 ceph-mon[24790]: osd.7 [v2:172.21.15.78:6824/3693180321,v1:172.21.15.78:6825/3693180321] boot 2024-05-20T22:19:00.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:59 smithi006 ceph-mon[24790]: osdmap e55: 8 total, 8 up, 8 in 2024-05-20T22:19:00.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:18:59 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:19:00.692 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1716243538,"num_in_osds":8,"osd_in_since":1716243522,"num_remapped_pgs":1} 2024-05-20T22:19:00.692 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T22:19:00.918 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:01.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:00 smithi078 ceph-mon[27225]: osdmap e56: 8 total, 8 up, 8 in 2024-05-20T22:19:01.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:00 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2664195354' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:19:01.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:00 smithi006 ceph-mon[24790]: osdmap e56: 8 total, 8 up, 8 in 2024-05-20T22:19:01.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:00 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2664195354' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-05-20T22:19:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: osdmap e57: 8 total, 8 up, 8 in 2024-05-20T22:19:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: Detected new or changed devices on smithi078 2024-05-20T22:19:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi078 to 3984M 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:19:02.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:02.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: osdmap e57: 8 total, 8 up, 8 in 2024-05-20T22:19:02.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: Detected new or changed devices on smithi078 2024-05-20T22:19:02.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi078 to 3984M 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:19:02.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:19:02.536 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:02.536 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":58,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","created":"2024-05-20T22:12:54.284291+0000","modified":"2024-05-20T22:19:01.936355+0000","last_up_change":"2024-05-20T22:18:58.917486+0000","last_in_change":"2024-05-20T22:18:42.013757+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T22:16:59.395085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"85a3ac26-fd4b-4838-8c9a-f936c3a4bc59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6803","nonce":750456943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6805","nonce":750456943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6809","nonce":750456943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6807","nonce":750456943}]},"public_addr":"172.21.15.6:6803/750456943","cluster_addr":"172.21.15.6:6805/750456943","heartbeat_back_addr":"172.21.15.6:6809/750456943","heartbeat_front_addr":"172.21.15.6:6807/750456943","state":["exists","up"]},{"osd":1,"uuid":"b3987af5-4f2c-4933-818b-5387358f701c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6811","nonce":3251516072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6813","nonce":3251516072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6817","nonce":3251516072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6815","nonce":3251516072}]},"public_addr":"172.21.15.6:6811/3251516072","cluster_addr":"172.21.15.6:6813/3251516072","heartbeat_back_addr":"172.21.15.6:6817/3251516072","heartbeat_front_addr":"172.21.15.6:6815/3251516072","state":["exists","up"]},{"osd":2,"uuid":"59140224-d800-4a50-81d7-711022859d52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6819","nonce":3612674218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6821","nonce":3612674218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6825","nonce":3612674218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6823","nonce":3612674218}]},"public_addr":"172.21.15.6:6819/3612674218","cluster_addr":"172.21.15.6:6821/3612674218","heartbeat_back_addr":"172.21.15.6:6825/3612674218","heartbeat_front_addr":"172.21.15.6:6823/3612674218","state":["exists","up"]},{"osd":3,"uuid":"d7f4845a-1617-47ab-be52-1efc2c8e3bca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6827","nonce":2017175948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6829","nonce":2017175948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6833","nonce":2017175948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6831","nonce":2017175948}]},"public_addr":"172.21.15.6:6827/2017175948","cluster_addr":"172.21.15.6:6829/2017175948","heartbeat_back_addr":"172.21.15.6:6833/2017175948","heartbeat_front_addr":"172.21.15.6:6831/2017175948","state":["exists","up"]},{"osd":4,"uuid":"906346ce-6f48-4cc2-8cd7-f8c03e431dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6801","nonce":1107461992}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6803","nonce":1107461992}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6807","nonce":1107461992}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6805","nonce":1107461992}]},"public_addr":"172.21.15.78:6801/1107461992","cluster_addr":"172.21.15.78:6803/1107461992","heartbeat_back_addr":"172.21.15.78:6807/1107461992","heartbeat_front_addr":"172.21.15.78:6805/1107461992","state":["exists","up"]},{"osd":5,"uuid":"02c0433e-9f55-4164-8316-92a948cd0e8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6809","nonce":476389303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6811","nonce":476389303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6815","nonce":476389303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6813","nonce":476389303}]},"public_addr":"172.21.15.78:6809/476389303","cluster_addr":"172.21.15.78:6811/476389303","heartbeat_back_addr":"172.21.15.78:6815/476389303","heartbeat_front_addr":"172.21.15.78:6813/476389303","state":["exists","up"]},{"osd":6,"uuid":"d4633543-b502-44cd-af23-96bdefbd1688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6817","nonce":2649823748}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6819","nonce":2649823748}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6823","nonce":2649823748}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6821","nonce":2649823748}]},"public_addr":"172.21.15.78:6817/2649823748","cluster_addr":"172.21.15.78:6819/2649823748","heartbeat_back_addr":"172.21.15.78:6823/2649823748","heartbeat_front_addr":"172.21.15.78:6821/2649823748","state":["exists","up"]},{"osd":7,"uuid":"4f06cc0b-609c-4b5f-945c-89cffe7b063a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6825","nonce":3693180321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6827","nonce":3693180321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6831","nonce":3693180321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6829","nonce":3693180321}]},"public_addr":"172.21.15.78:6825/3693180321","cluster_addr":"172.21.15.78:6827/3693180321","heartbeat_back_addr":"172.21.15.78:6831/3693180321","heartbeat_front_addr":"172.21.15.78:6829/3693180321","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:15:58.728185+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:16:28.317833+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:16:56.532948+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:17:23.771937+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:17:45.254356+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:08.738816+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:33.041359+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:57.863591+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:6801/320801600":"2024-05-21T22:14:54.825946+0000","172.21.15.6:6800/320801600":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/1775284158":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/2279419643":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/2873584017":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/2415967146":"2024-05-21T22:13:58.147539+0000","172.21.15.6:6801/1540556595":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/971610074":"2024-05-21T22:13:58.147539+0000","172.21.15.6:6800/1540556595":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/3189118011":"2024-05-21T22:13:27.824794+0000","172.21.15.6:6800/3207974289":"2024-05-21T22:13:27.824794+0000","172.21.15.6:0/1676395519":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/3957443762":"2024-05-21T22:13:27.824794+0000","172.21.15.6:0/889489299":"2024-05-21T22:13:27.824794+0000","172.21.15.6:6801/3207974289":"2024-05-21T22:13:27.824794+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T22:19:03.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:02 smithi006 ceph-mon[24790]: osdmap e58: 8 total, 8 up, 8 in 2024-05-20T22:19:03.173 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:02 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1469379545' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:19:03.200 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-05-20T22:16:59.395085+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-05-20T22:19:03.200 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-05-20T22:19:03.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:02 smithi078 ceph-mon[27225]: osdmap e58: 8 total, 8 up, 8 in 2024-05-20T22:19:03.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:02 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1469379545' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:19:03.425 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:04.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:03 smithi006 ceph-mon[24790]: pgmap v170: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:19:04.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:03 smithi006 ceph-mon[24790]: osdmap e59: 8 total, 8 up, 8 in 2024-05-20T22:19:04.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:03 smithi078 ceph-mon[27225]: pgmap v170: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-05-20T22:19:04.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:03 smithi078 ceph-mon[27225]: osdmap e59: 8 total, 8 up, 8 in 2024-05-20T22:19:05.253 INFO:teuthology.orchestra.run.smithi006.stdout:pg_num: 1 2024-05-20T22:19:05.957 INFO:tasks.cephadm:Setting up client nodes... 2024-05-20T22:19:05.957 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-05-20T22:19:06.275 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:05 smithi006 ceph-mon[24790]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-05-20T22:19:06.275 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:05 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3125101039' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T22:19:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:05 smithi078 ceph-mon[27225]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-05-20T22:19:06.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:05 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/3125101039' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-05-20T22:19:08.031 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:07 smithi006 ceph-mon[24790]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-20T22:19:08.203 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:07 smithi078 ceph-mon[27225]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-05-20T22:19:08.314 INFO:teuthology.orchestra.run.smithi006.stdout:[client.0] 2024-05-20T22:19:08.314 INFO:teuthology.orchestra.run.smithi006.stdout: key = AQBczEtmLNmDEhAAufqN7PDwo1XWjyeeLmDj/g== 2024-05-20T22:19:08.995 DEBUG:teuthology.orchestra.run.smithi006:> set -ex 2024-05-20T22:19:08.995 DEBUG:teuthology.orchestra.run.smithi006:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-05-20T22:19:08.995 DEBUG:teuthology.orchestra.run.smithi006:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-05-20T22:19:09.036 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-05-20T22:19:09.036 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-05-20T22:19:09.037 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph mgr dump --format=json 2024-05-20T22:19:09.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:08 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1893058606' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:19:09.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:08 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1893058606' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:19:09.297 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:09.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:08 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1893058606' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-05-20T22:19:09.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:08 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1893058606' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-05-20T22:19:10.619 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:10 smithi006 ceph-mon[24790]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-20T22:19:10.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:10 smithi078 ceph-mon[27225]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-05-20T22:19:11.180 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:11.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:11 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/4262557508' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:19:11.933 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:11 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/4262557508' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-05-20T22:19:11.960 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":19,"active_gid":14206,"active_name":"smithi006.ssyogp","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6800","nonce":885170914},{"type":"v1","addr":"172.21.15.6:6801","nonce":885170914}]},"active_addr":"172.21.15.6:6801/885170914","active_change":"2024-05-20T22:14:54.826250+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14228,"name":"smithi078.vxzcbt","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.6:8443/","prometheus":"http://172.21.15.6:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":2032724621}]},{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":4028866067}]},{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":1484688546}]},{"addrvec":[{"type":"v2","addr":"172.21.15.6:0","nonce":966542097}]}]}} 2024-05-20T22:19:11.965 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-05-20T22:19:11.965 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-05-20T22:19:11.965 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T22:19:12.191 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:12.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:12 smithi078 ceph-mon[27225]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:12.949 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:12 smithi006 ceph-mon[24790]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:13.998 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:13.998 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":59,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","created":"2024-05-20T22:12:54.284291+0000","modified":"2024-05-20T22:19:02.941368+0000","last_up_change":"2024-05-20T22:18:58.917486+0000","last_in_change":"2024-05-20T22:18:42.013757+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T22:16:59.395085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"85a3ac26-fd4b-4838-8c9a-f936c3a4bc59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6803","nonce":750456943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6805","nonce":750456943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6809","nonce":750456943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6807","nonce":750456943}]},"public_addr":"172.21.15.6:6803/750456943","cluster_addr":"172.21.15.6:6805/750456943","heartbeat_back_addr":"172.21.15.6:6809/750456943","heartbeat_front_addr":"172.21.15.6:6807/750456943","state":["exists","up"]},{"osd":1,"uuid":"b3987af5-4f2c-4933-818b-5387358f701c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6811","nonce":3251516072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6813","nonce":3251516072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6817","nonce":3251516072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6815","nonce":3251516072}]},"public_addr":"172.21.15.6:6811/3251516072","cluster_addr":"172.21.15.6:6813/3251516072","heartbeat_back_addr":"172.21.15.6:6817/3251516072","heartbeat_front_addr":"172.21.15.6:6815/3251516072","state":["exists","up"]},{"osd":2,"uuid":"59140224-d800-4a50-81d7-711022859d52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6819","nonce":3612674218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6821","nonce":3612674218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6825","nonce":3612674218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6823","nonce":3612674218}]},"public_addr":"172.21.15.6:6819/3612674218","cluster_addr":"172.21.15.6:6821/3612674218","heartbeat_back_addr":"172.21.15.6:6825/3612674218","heartbeat_front_addr":"172.21.15.6:6823/3612674218","state":["exists","up"]},{"osd":3,"uuid":"d7f4845a-1617-47ab-be52-1efc2c8e3bca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6827","nonce":2017175948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6829","nonce":2017175948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6833","nonce":2017175948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6831","nonce":2017175948}]},"public_addr":"172.21.15.6:6827/2017175948","cluster_addr":"172.21.15.6:6829/2017175948","heartbeat_back_addr":"172.21.15.6:6833/2017175948","heartbeat_front_addr":"172.21.15.6:6831/2017175948","state":["exists","up"]},{"osd":4,"uuid":"906346ce-6f48-4cc2-8cd7-f8c03e431dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6801","nonce":1107461992}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6803","nonce":1107461992}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6807","nonce":1107461992}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6805","nonce":1107461992}]},"public_addr":"172.21.15.78:6801/1107461992","cluster_addr":"172.21.15.78:6803/1107461992","heartbeat_back_addr":"172.21.15.78:6807/1107461992","heartbeat_front_addr":"172.21.15.78:6805/1107461992","state":["exists","up"]},{"osd":5,"uuid":"02c0433e-9f55-4164-8316-92a948cd0e8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6809","nonce":476389303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6811","nonce":476389303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6815","nonce":476389303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6813","nonce":476389303}]},"public_addr":"172.21.15.78:6809/476389303","cluster_addr":"172.21.15.78:6811/476389303","heartbeat_back_addr":"172.21.15.78:6815/476389303","heartbeat_front_addr":"172.21.15.78:6813/476389303","state":["exists","up"]},{"osd":6,"uuid":"d4633543-b502-44cd-af23-96bdefbd1688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6817","nonce":2649823748}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6819","nonce":2649823748}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6823","nonce":2649823748}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6821","nonce":2649823748}]},"public_addr":"172.21.15.78:6817/2649823748","cluster_addr":"172.21.15.78:6819/2649823748","heartbeat_back_addr":"172.21.15.78:6823/2649823748","heartbeat_front_addr":"172.21.15.78:6821/2649823748","state":["exists","up"]},{"osd":7,"uuid":"4f06cc0b-609c-4b5f-945c-89cffe7b063a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6825","nonce":3693180321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6827","nonce":3693180321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6831","nonce":3693180321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6829","nonce":3693180321}]},"public_addr":"172.21.15.78:6825/3693180321","cluster_addr":"172.21.15.78:6827/3693180321","heartbeat_back_addr":"172.21.15.78:6831/3693180321","heartbeat_front_addr":"172.21.15.78:6829/3693180321","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:15:58.728185+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:16:28.317833+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:16:56.532948+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:17:23.771937+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:17:45.254356+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:08.738816+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:33.041359+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:57.863591+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:6801/320801600":"2024-05-21T22:14:54.825946+0000","172.21.15.6:6800/320801600":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/1775284158":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/2279419643":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/2873584017":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/2415967146":"2024-05-21T22:13:58.147539+0000","172.21.15.6:6801/1540556595":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/971610074":"2024-05-21T22:13:58.147539+0000","172.21.15.6:6800/1540556595":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/3189118011":"2024-05-21T22:13:27.824794+0000","172.21.15.6:6800/3207974289":"2024-05-21T22:13:27.824794+0000","172.21.15.6:0/1676395519":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/3957443762":"2024-05-21T22:13:27.824794+0000","172.21.15.6:0/889489299":"2024-05-21T22:13:27.824794+0000","172.21.15.6:6801/3207974289":"2024-05-21T22:13:27.824794+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T22:19:14.697 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-05-20T22:19:14.697 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd dump --format=json 2024-05-20T22:19:14.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:14 smithi078 ceph-mon[27225]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:14.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:14 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1303847555' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:19:14.925 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:14 smithi006 ceph-mon[24790]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:15.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:14 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1303847555' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:19:16.733 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:16.733 INFO:teuthology.orchestra.run.smithi006.stdout:{"epoch":59,"fsid":"00e42188-16f6-11ef-bc9a-c7b262605968","created":"2024-05-20T22:12:54.284291+0000","modified":"2024-05-20T22:19:02.941368+0000","last_up_change":"2024-05-20T22:18:58.917486+0000","last_in_change":"2024-05-20T22:18:42.013757+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-05-20T22:16:59.395085+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"85a3ac26-fd4b-4838-8c9a-f936c3a4bc59","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6802","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6803","nonce":750456943}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6804","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6805","nonce":750456943}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6808","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6809","nonce":750456943}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6806","nonce":750456943},{"type":"v1","addr":"172.21.15.6:6807","nonce":750456943}]},"public_addr":"172.21.15.6:6803/750456943","cluster_addr":"172.21.15.6:6805/750456943","heartbeat_back_addr":"172.21.15.6:6809/750456943","heartbeat_front_addr":"172.21.15.6:6807/750456943","state":["exists","up"]},{"osd":1,"uuid":"b3987af5-4f2c-4933-818b-5387358f701c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6810","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6811","nonce":3251516072}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6812","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6813","nonce":3251516072}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6816","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6817","nonce":3251516072}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6814","nonce":3251516072},{"type":"v1","addr":"172.21.15.6:6815","nonce":3251516072}]},"public_addr":"172.21.15.6:6811/3251516072","cluster_addr":"172.21.15.6:6813/3251516072","heartbeat_back_addr":"172.21.15.6:6817/3251516072","heartbeat_front_addr":"172.21.15.6:6815/3251516072","state":["exists","up"]},{"osd":2,"uuid":"59140224-d800-4a50-81d7-711022859d52","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6818","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6819","nonce":3612674218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6820","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6821","nonce":3612674218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6824","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6825","nonce":3612674218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6822","nonce":3612674218},{"type":"v1","addr":"172.21.15.6:6823","nonce":3612674218}]},"public_addr":"172.21.15.6:6819/3612674218","cluster_addr":"172.21.15.6:6821/3612674218","heartbeat_back_addr":"172.21.15.6:6825/3612674218","heartbeat_front_addr":"172.21.15.6:6823/3612674218","state":["exists","up"]},{"osd":3,"uuid":"d7f4845a-1617-47ab-be52-1efc2c8e3bca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6826","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6827","nonce":2017175948}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6828","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6829","nonce":2017175948}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6832","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6833","nonce":2017175948}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.6:6830","nonce":2017175948},{"type":"v1","addr":"172.21.15.6:6831","nonce":2017175948}]},"public_addr":"172.21.15.6:6827/2017175948","cluster_addr":"172.21.15.6:6829/2017175948","heartbeat_back_addr":"172.21.15.6:6833/2017175948","heartbeat_front_addr":"172.21.15.6:6831/2017175948","state":["exists","up"]},{"osd":4,"uuid":"906346ce-6f48-4cc2-8cd7-f8c03e431dcb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6800","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6801","nonce":1107461992}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6802","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6803","nonce":1107461992}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6806","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6807","nonce":1107461992}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6804","nonce":1107461992},{"type":"v1","addr":"172.21.15.78:6805","nonce":1107461992}]},"public_addr":"172.21.15.78:6801/1107461992","cluster_addr":"172.21.15.78:6803/1107461992","heartbeat_back_addr":"172.21.15.78:6807/1107461992","heartbeat_front_addr":"172.21.15.78:6805/1107461992","state":["exists","up"]},{"osd":5,"uuid":"02c0433e-9f55-4164-8316-92a948cd0e8f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6808","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6809","nonce":476389303}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6810","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6811","nonce":476389303}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6814","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6815","nonce":476389303}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6812","nonce":476389303},{"type":"v1","addr":"172.21.15.78:6813","nonce":476389303}]},"public_addr":"172.21.15.78:6809/476389303","cluster_addr":"172.21.15.78:6811/476389303","heartbeat_back_addr":"172.21.15.78:6815/476389303","heartbeat_front_addr":"172.21.15.78:6813/476389303","state":["exists","up"]},{"osd":6,"uuid":"d4633543-b502-44cd-af23-96bdefbd1688","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6816","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6817","nonce":2649823748}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6818","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6819","nonce":2649823748}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6822","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6823","nonce":2649823748}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6820","nonce":2649823748},{"type":"v1","addr":"172.21.15.78:6821","nonce":2649823748}]},"public_addr":"172.21.15.78:6817/2649823748","cluster_addr":"172.21.15.78:6819/2649823748","heartbeat_back_addr":"172.21.15.78:6823/2649823748","heartbeat_front_addr":"172.21.15.78:6821/2649823748","state":["exists","up"]},{"osd":7,"uuid":"4f06cc0b-609c-4b5f-945c-89cffe7b063a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6824","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6825","nonce":3693180321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6826","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6827","nonce":3693180321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6830","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6831","nonce":3693180321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.78:6828","nonce":3693180321},{"type":"v1","addr":"172.21.15.78:6829","nonce":3693180321}]},"public_addr":"172.21.15.78:6825/3693180321","cluster_addr":"172.21.15.78:6827/3693180321","heartbeat_back_addr":"172.21.15.78:6831/3693180321","heartbeat_front_addr":"172.21.15.78:6829/3693180321","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:15:58.728185+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:16:28.317833+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:16:56.532948+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:17:23.771937+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:17:45.254356+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:08.738816+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:33.041359+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-05-20T22:18:57.863591+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.6:6801/320801600":"2024-05-21T22:14:54.825946+0000","172.21.15.6:6800/320801600":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/1775284158":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/2279419643":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/2873584017":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/2415967146":"2024-05-21T22:13:58.147539+0000","172.21.15.6:6801/1540556595":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/971610074":"2024-05-21T22:13:58.147539+0000","172.21.15.6:6800/1540556595":"2024-05-21T22:13:58.147539+0000","172.21.15.6:0/3189118011":"2024-05-21T22:13:27.824794+0000","172.21.15.6:6800/3207974289":"2024-05-21T22:13:27.824794+0000","172.21.15.6:0/1676395519":"2024-05-21T22:14:54.825946+0000","172.21.15.6:0/3957443762":"2024-05-21T22:13:27.824794+0000","172.21.15.6:0/889489299":"2024-05-21T22:13:27.824794+0000","172.21.15.6:6801/3207974289":"2024-05-21T22:13:27.824794+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-05-20T22:19:16.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:16 smithi078 ceph-mon[27225]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:17.028 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:16 smithi006 ceph-mon[24790]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:17.417 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-05-20T22:19:17.418 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-05-20T22:19:17.418 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-05-20T22:19:17.418 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-05-20T22:19:17.419 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-05-20T22:19:17.419 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-05-20T22:19:17.420 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-05-20T22:19:17.420 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-05-20T22:19:17.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:17 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2052793466' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:19:17.856 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.862 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.864 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.864 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.866 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.866 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.871 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.871 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:17.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:17 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2052793466' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-05-20T22:19:18.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:18 smithi078 ceph-mon[27225]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:19.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:18 smithi006 ceph-mon[24790]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:19.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:19 smithi078 ceph-mon[27225]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:19.927 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:19 smithi006 ceph-mon[24790]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:22.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:22 smithi006 ceph-mon[24790]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:22.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:22 smithi078 ceph-mon[27225]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:24.720 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:24 smithi006 ceph-mon[24790]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:24.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:24 smithi078 ceph-mon[27225]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:26.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:26 smithi006 ceph-mon[24790]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:26.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:26 smithi078 ceph-mon[27225]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:28.422 INFO:teuthology.orchestra.run.smithi006.stdout:38654705707 2024-05-20T22:19:28.423 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-05-20T22:19:28.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:28 smithi006 ceph-mon[24790]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:28.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:28 smithi078 ceph-mon[27225]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:29.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:29 smithi006 ceph-mon[24790]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:29.802 INFO:teuthology.orchestra.run.smithi006.stdout:81604378655 2024-05-20T22:19:29.803 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-05-20T22:19:29.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:29 smithi078 ceph-mon[27225]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:30.582 INFO:teuthology.orchestra.run.smithi006.stdout:133143986199 2024-05-20T22:19:30.583 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-05-20T22:19:31.689 INFO:teuthology.orchestra.run.smithi006.stdout:201863462925 2024-05-20T22:19:31.689 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-05-20T22:19:32.095 INFO:teuthology.orchestra.run.smithi006.stdout:111669149724 2024-05-20T22:19:32.095 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-05-20T22:19:32.410 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:32.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:32 smithi006 ceph-mon[24790]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:32.840 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:32 smithi078 ceph-mon[27225]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:32.873 INFO:teuthology.orchestra.run.smithi006.stdout:60129542183 2024-05-20T22:19:32.874 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-05-20T22:19:33.259 INFO:teuthology.orchestra.run.smithi006.stdout:236223201289 2024-05-20T22:19:33.259 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-05-20T22:19:34.109 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:34.109 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:34.109 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:34.110 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:34.119 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:34.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:34 smithi006 ceph-mon[24790]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:34.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:34 smithi078 ceph-mon[27225]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:35.648 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:36.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:36 smithi006 ceph-mon[24790]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:36.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:36 smithi078 ceph-mon[27225]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:38.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:38 smithi078 ceph-mon[27225]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:39.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:38 smithi006 ceph-mon[24790]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:39.082 INFO:teuthology.orchestra.run.smithi006.stdout:167503724563 2024-05-20T22:19:39.083 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-05-20T22:19:39.634 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:39 smithi006 ceph-mon[24790]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:39.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:39 smithi078 ceph-mon[27225]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:41.174 INFO:teuthology.orchestra.run.smithi006.stdout:111669149726 2024-05-20T22:19:41.582 INFO:teuthology.orchestra.run.smithi006.stdout:201863462928 2024-05-20T22:19:41.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:41 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2030527634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T22:19:41.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:41 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2030527634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-05-20T22:19:42.607 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:42 smithi006 ceph-mon[24790]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:42.608 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:42 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3313483299' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T22:19:42.839 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:42 smithi078 ceph-mon[27225]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:42.839 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:42 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/3313483299' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-05-20T22:19:43.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:43 smithi006 ceph-mon[24790]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:43.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:43 smithi078 ceph-mon[27225]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:44.646 INFO:teuthology.orchestra.run.smithi006.stdout:60129542186 2024-05-20T22:19:44.838 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:44 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1203121790' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T22:19:44.861 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462925 got 201863462928 for osd.6 2024-05-20T22:19:44.862 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:44.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:44 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1203121790' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-05-20T22:19:45.233 INFO:teuthology.orchestra.run.smithi006.stdout:81604378659 2024-05-20T22:19:45.867 INFO:teuthology.orchestra.run.smithi006.stdout:38654705711 2024-05-20T22:19:45.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:45 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/424186356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T22:19:45.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:45 smithi006 ceph-mon[24790]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:45.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:45 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1130638499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T22:19:45.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:45 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/424186356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-05-20T22:19:45.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:45 smithi078 ceph-mon[27225]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:45.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:45 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1130638499' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-05-20T22:19:46.095 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378655 got 81604378659 for osd.2 2024-05-20T22:19:46.095 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:46.171 INFO:teuthology.orchestra.run.smithi006.stdout:133143986202 2024-05-20T22:19:46.746 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986199 got 133143986202 for osd.4 2024-05-20T22:19:46.746 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:46.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:46 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1273532636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T22:19:47.020 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:46 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1273532636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-05-20T22:19:47.316 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542186 for osd.1 2024-05-20T22:19:47.317 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:47.709 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149724 got 111669149726 for osd.3 2024-05-20T22:19:47.710 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:47.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:47 smithi078 ceph-mon[27225]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:47.911 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:47 smithi006 ceph-mon[24790]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:48.110 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:48.532 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705707 got 38654705711 for osd.0 2024-05-20T22:19:48.532 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:48.999 INFO:teuthology.orchestra.run.smithi006.stdout:236223201292 2024-05-20T22:19:50.051 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-05-20T22:19:50.051 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:50.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:49 smithi006 ceph-mon[24790]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:50.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:49 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/4203534787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T22:19:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:49 smithi078 ceph-mon[27225]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:50.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:49 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/4203534787' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-05-20T22:19:51.128 INFO:teuthology.orchestra.run.smithi006.stdout:167503724566 2024-05-20T22:19:51.758 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:51 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1715197261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T22:19:51.784 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724563 got 167503724566 for osd.5 2024-05-20T22:19:51.784 DEBUG:teuthology.parallel:result is None 2024-05-20T22:19:51.785 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-05-20T22:19:51.785 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T22:19:51.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:51 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1715197261' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-05-20T22:19:52.013 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:52.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:52 smithi006 ceph-mon[24790]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:52.839 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:52 smithi078 ceph-mon[27225]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:53.763 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:53.764 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2024-05-20T22:19:54.412 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":196,"stamp":"2024-05-20T22:19:52.932098+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48880,"kb_used_data":6320,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912336,"statfs":{"total":767960285184,"available":767910232064,"internally_reserved":0,"allocated":6471680,"data_stored":3548118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001846"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-05-20T22:19:02.948536+0000","last_change":"2024-05-20T22:19:02.948536+0000","last_active":"2024-05-20T22:19:02.948536+0000","last_peered":"2024-05-20T22:19:02.948536+0000","last_clean":"2024-05-20T22:19:02.948536+0000","last_became_active":"2024-05-20T22:19:02.948140+0000","last_became_peered":"2024-05-20T22:19:02.948140+0000","last_unstale":"2024-05-20T22:19:02.948536+0000","last_undegraded":"2024-05-20T22:19:02.948536+0000","last_fullsized":"2024-05-20T22:19:02.948536+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T22:16:59.561374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T22:16:59.561374+0000","last_clean_scrub_stamp":"2024-05-20T22:16:59.561374+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:34:03.454196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67600000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66200000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.82599999999999996}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63600000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61499999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.6970000000000001,"5min":6.6970000000000001,"15min":6.6970000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.48099999999999998}]},{"osd":1,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.7119999999999997,"5min":6.7119999999999997,"15min":6.7119999999999997},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.628}]},{"osd":2,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.499},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.61499999999999999}]},{"osd":3,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.56899999999999995}]},{"osd":4,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.38100000000000001}]},{"osd":5,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.42799999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48699999999999999}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.34999999999999998,"15min":0.32800000000000001},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.36899999999999999,"15min":0.34599999999999997},"min":{"1min":0.31,"5min":0.254,"15min":0.254},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.312}]},{"osd":2,"last update":"Mon May 20 22:19:02 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.38100000000000001,"15min":0.374},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.377},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.39100000000000001,"15min":0.38600000000000001},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon May 20 22:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.61199999999999999,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.46800000000000003,"15min":0.46400000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon May 20 22:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58199999999999996,"15min":0.57799999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.438},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58799999999999997,"15min":0.59299999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon May 20 22:19:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.63400000000000001}]},{"osd":6,"last update":"Mon May 20 22:19:39 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.113,"5min":1.113,"15min":1.113},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.53100000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon May 20 22:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.34599999999999997,"15min":0.32000000000000001},"min":{"1min":0.26800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.35999999999999999,"15min":0.33200000000000002},"min":{"1min":0.27500000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.58699999999999997,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.45900000000000002}]},{"osd":2,"last update":"Mon May 20 22:19:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.371,"15min":0.35699999999999998},"min":{"1min":0.29399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.39900000000000002,"15min":0.38300000000000001},"min":{"1min":0.38600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.50700000000000001}]},{"osd":3,"last update":"Mon May 20 22:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46600000000000003,"15min":0.46800000000000003},"min":{"1min":0.33300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.373,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.66800000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.61099999999999999}]},{"osd":4,"last update":"Mon May 20 22:19:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.621,"15min":0.61899999999999999},"min":{"1min":0.42199999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.628,"15min":0.625},"min":{"1min":0.442,"5min":0.379,"15min":0.379},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.56100000000000005}]},{"osd":5,"last update":"Mon May 20 22:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.59599999999999997}]},{"osd":6,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.67200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.627},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.39400000000000002,"15min":0.38200000000000001},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.223},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.505}]},{"osd":1,"last update":"Mon May 20 22:19:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.41399999999999998,"15min":0.40200000000000002},"min":{"1min":0.29299999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.433,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.28899999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.45900000000000002}]},{"osd":3,"last update":"Mon May 20 22:19:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.44700000000000001,"15min":0.45100000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.44},{"interface":"front","average":{"1min":0.502,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.58199999999999996}]},{"osd":4,"last update":"Mon May 20 22:18:50 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.54600000000000004}]},{"osd":5,"last update":"Mon May 20 22:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":18.576000000000001,"5min":18.576000000000001,"15min":18.576000000000001},"last":0.68000000000000005}]},{"osd":6,"last update":"Mon May 20 22:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.59699999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65900000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.40100000000000002,"15min":0.39800000000000002},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.41699999999999998,"15min":0.41399999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.30199999999999999}]},{"osd":1,"last update":"Mon May 20 22:19:29 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.28799999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.749,"15min":0.749},"last":0.249},{"interface":"front","average":{"1min":0.435,"5min":0.439,"15min":0.44},"min":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.67800000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon May 20 22:19:29 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.52100000000000002,"15min":0.52400000000000002},"min":{"1min":0.35399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.73899999999999999,"5min":0.877,"15min":0.877},"last":0.316},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48399999999999999,"15min":0.48799999999999999},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon May 20 22:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84999999999999998,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.629},{"interface":"front","average":{"1min":0.873,"5min":0.66100000000000003,"15min":0.625},"min":{"1min":0.441,"5min":0.374,"15min":0.374},"max":{"1min":4.6059999999999999,"5min":4.6059999999999999,"15min":4.6059999999999999},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon May 20 22:19:12 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.69499999999999995}]},{"osd":6,"last update":"Mon May 20 22:19:35 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.624},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.73399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:18:49 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.57599999999999996}]},{"osd":1,"last update":"Mon May 20 22:18:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.876,"5min":0.876,"15min":0.876},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.67700000000000005}]},{"osd":2,"last update":"Mon May 20 22:18:49 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.52800000000000002}]},{"osd":3,"last update":"Mon May 20 22:18:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.56299999999999994}]},{"osd":5,"last update":"Mon May 20 22:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.61299999999999999}]},{"osd":6,"last update":"Mon May 20 22:19:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64300000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724567,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.45900000000000002}]},{"osd":1,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.437}]},{"osd":2,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.443},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.502},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.496}]},{"osd":4,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon May 20 22:19:38 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.39000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T22:19:54.414 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph pg dump --format=json 2024-05-20T22:19:54.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:54 smithi006 ceph-mon[24790]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:54.640 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:54.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:54 smithi078 ceph-mon[27225]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:55.685 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:55 smithi006 ceph-mon[24790]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:19:55.685 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:19:55.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:19:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:55 smithi078 ceph-mon[27225]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:19:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:19:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:19:56.423 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:56.425 INFO:teuthology.orchestra.run.smithi006.stderr:dumped all 2024-05-20T22:19:56.726 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:56 smithi006 ceph-mon[24790]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:56.726 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:56 smithi006 ceph-mon[24790]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:19:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:56 smithi078 ceph-mon[27225]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:56 smithi078 ceph-mon[27225]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:19:57.054 INFO:teuthology.orchestra.run.smithi006.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-05-20T22:19:54.932368+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48880,"kb_used_data":6320,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912336,"statfs":{"total":767960285184,"available":767910232064,"internally_reserved":0,"allocated":6471680,"data_stored":3548118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001797"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-05-20T22:19:02.948536+0000","last_change":"2024-05-20T22:19:02.948536+0000","last_active":"2024-05-20T22:19:02.948536+0000","last_peered":"2024-05-20T22:19:02.948536+0000","last_clean":"2024-05-20T22:19:02.948536+0000","last_became_active":"2024-05-20T22:19:02.948140+0000","last_became_peered":"2024-05-20T22:19:02.948140+0000","last_unstale":"2024-05-20T22:19:02.948536+0000","last_undegraded":"2024-05-20T22:19:02.948536+0000","last_fullsized":"2024-05-20T22:19:02.948536+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-05-20T22:16:59.561374+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-05-20T22:16:59.561374+0000","last_clean_scrub_stamp":"2024-05-20T22:16:59.561374+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-22T01:34:03.454196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41699999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.5}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45600000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36599999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57499999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.6970000000000001,"5min":6.6970000000000001,"15min":6.6970000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.57899999999999996}]},{"osd":1,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.7119999999999997,"5min":6.7119999999999997,"15min":6.7119999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.60999999999999999}]},{"osd":2,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.504}]},{"osd":3,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.629}]},{"osd":4,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.378},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.434}]},{"osd":5,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.49299999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39700000000000002}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:39 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.34999999999999998,"15min":0.32800000000000001},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.36899999999999999,"15min":0.34599999999999997},"min":{"1min":0.31,"5min":0.254,"15min":0.254},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.312}]},{"osd":2,"last update":"Mon May 20 22:19:02 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.38100000000000001,"15min":0.374},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.377},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.39100000000000001,"15min":0.38600000000000001},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.29799999999999999}]},{"osd":3,"last update":"Mon May 20 22:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.61199999999999999,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.496,"5min":0.46800000000000003,"15min":0.46400000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.42199999999999999}]},{"osd":4,"last update":"Mon May 20 22:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58199999999999996,"15min":0.57799999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.438},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58799999999999997,"15min":0.59299999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.47599999999999998}]},{"osd":5,"last update":"Mon May 20 22:19:09 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.63400000000000001}]},{"osd":6,"last update":"Mon May 20 22:19:39 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.113,"5min":1.113,"15min":1.113},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.53100000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon May 20 22:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.34599999999999997,"15min":0.32000000000000001},"min":{"1min":0.26800000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.35999999999999999,"15min":0.33200000000000002},"min":{"1min":0.27500000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.58699999999999997,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.45900000000000002}]},{"osd":2,"last update":"Mon May 20 22:19:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.371,"15min":0.35699999999999998},"min":{"1min":0.29399999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.39900000000000002,"15min":0.38300000000000001},"min":{"1min":0.38600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.50700000000000001}]},{"osd":3,"last update":"Mon May 20 22:19:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.46600000000000003,"15min":0.46800000000000003},"min":{"1min":0.33300000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.373,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.66800000000000004,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.61099999999999999}]},{"osd":4,"last update":"Mon May 20 22:19:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.621,"15min":0.61899999999999999},"min":{"1min":0.42199999999999999,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.99399999999999999,"5min":0.99399999999999999,"15min":0.99399999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.628,"15min":0.625},"min":{"1min":0.442,"5min":0.379,"15min":0.379},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.56100000000000005}]},{"osd":5,"last update":"Mon May 20 22:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.59599999999999997}]},{"osd":6,"last update":"Mon May 20 22:19:37 2024","interfaces":[{"interface":"back","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0509999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.67200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:04 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.371,"15min":0.36599999999999999},"min":{"1min":0.23499999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.39400000000000002,"15min":0.38200000000000001},"min":{"1min":0.29099999999999998,"5min":0.223,"15min":0.223},"max":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"last":0.54500000000000004}]},{"osd":1,"last update":"Mon May 20 22:19:04 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.41399999999999998,"15min":0.40200000000000002},"min":{"1min":0.29299999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.433,"5min":0.39200000000000002,"15min":0.38500000000000001},"min":{"1min":0.28899999999999998,"5min":0.222,"15min":0.222},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.52000000000000002}]},{"osd":3,"last update":"Mon May 20 22:19:30 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.44700000000000001,"15min":0.45100000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.51400000000000001,"15min":0.51600000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.58099999999999996}]},{"osd":4,"last update":"Mon May 20 22:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.63200000000000001,"15min":0.63800000000000001},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.44600000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.78000000000000003,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.76700000000000002}]},{"osd":5,"last update":"Mon May 20 22:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.80700000000000005},{"interface":"front","average":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":18.576000000000001,"5min":18.576000000000001,"15min":18.576000000000001},"last":0.82299999999999995}]},{"osd":6,"last update":"Mon May 20 22:19:34 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.89500000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.79400000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77600000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.40100000000000002,"15min":0.39800000000000002},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.41699999999999998,"15min":0.41399999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.30199999999999999}]},{"osd":1,"last update":"Mon May 20 22:19:29 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.43099999999999999,"15min":0.43099999999999999},"min":{"1min":0.28799999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63500000000000001,"5min":0.749,"15min":0.749},"last":0.249},{"interface":"front","average":{"1min":0.435,"5min":0.439,"15min":0.44},"min":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.67800000000000005,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.27400000000000002}]},{"osd":2,"last update":"Mon May 20 22:19:29 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.52100000000000002,"15min":0.52400000000000002},"min":{"1min":0.35399999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.73899999999999999,"5min":0.877,"15min":0.877},"last":0.316},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.48399999999999999,"15min":0.48799999999999999},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38100000000000001}]},{"osd":4,"last update":"Mon May 20 22:19:51 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.58499999999999996,"15min":0.58899999999999997},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.84999999999999998,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.629},{"interface":"front","average":{"1min":0.873,"5min":0.66100000000000003,"15min":0.625},"min":{"1min":0.441,"5min":0.374,"15min":0.374},"max":{"1min":4.6059999999999999,"5min":4.6059999999999999,"15min":4.6059999999999999},"last":0.52000000000000002}]},{"osd":5,"last update":"Mon May 20 22:19:12 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.69499999999999995}]},{"osd":6,"last update":"Mon May 20 22:19:35 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.624},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.73399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.373,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.68300000000000005,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.61099999999999999,"15min":0.57899999999999996},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.1619999999999999,"5min":7.1619999999999999,"15min":7.1619999999999999},"last":0.47099999999999997}]},{"osd":1,"last update":"Mon May 20 22:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.55700000000000005,"15min":0.56299999999999994},"min":{"1min":0.39100000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.61199999999999999,"5min":0.876,"15min":0.876},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.56499999999999995,"15min":0.57099999999999995},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.67700000000000005,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48199999999999998}]},{"osd":2,"last update":"Mon May 20 22:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.55000000000000004,"15min":0.55200000000000005},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.70499999999999996,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.376},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.54600000000000004,"15min":0.55500000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69299999999999995,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.33300000000000002}]},{"osd":3,"last update":"Mon May 20 22:19:50 2024","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.63300000000000001,"15min":0.60499999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.1029999999999998,"5min":7.1029999999999998,"15min":7.1029999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.67500000000000004,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.49399999999999999}]},{"osd":5,"last update":"Mon May 20 22:19:11 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.373,"15min":0.373},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.438}]},{"osd":6,"last update":"Mon May 20 22:19:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.51400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46200000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724567,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.45900000000000002}]},{"osd":1,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.437}]},{"osd":2,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.443},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.51400000000000001}]},{"osd":3,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.502},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.496}]},{"osd":4,"last update":"Mon May 20 22:19:13 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.36599999999999999}]},{"osd":6,"last update":"Mon May 20 22:19:38 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.39000000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-05-20T22:19:57.055 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-05-20T22:19:57.055 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-05-20T22:19:57.055 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-05-20T22:19:57.056 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph health --format=json 2024-05-20T22:19:57.280 INFO:teuthology.orchestra.run.smithi006.stderr:Inferring config /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/mon.smithi006/config 2024-05-20T22:19:57.713 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:57 smithi006 ceph-mon[24790]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:57.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:57 smithi078 ceph-mon[27225]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:19:58.999 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:19:58.999 INFO:teuthology.orchestra.run.smithi006.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-05-20T22:19:59.618 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-05-20T22:19:59.619 INFO:tasks.cephadm:Setup complete, yielding 2024-05-20T22:19:59.619 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T22:19:59.628 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:19:59.628 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch status' 2024-05-20T22:20:00.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:59 smithi006 ceph-mon[24790]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:00.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:19:59 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/341352437' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T22:20:00.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:59 smithi078 ceph-mon[27225]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:00.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:19:59 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/341352437' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-05-20T22:20:01.255 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:00 smithi006 ceph-mon[24790]: overall HEALTH_OK 2024-05-20T22:20:01.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:00 smithi078 ceph-mon[27225]: overall HEALTH_OK 2024-05-20T22:20:01.734 INFO:teuthology.orchestra.run.smithi006.stdout:Backend: cephadm 2024-05-20T22:20:01.735 INFO:teuthology.orchestra.run.smithi006.stdout:Available: Yes 2024-05-20T22:20:01.735 INFO:teuthology.orchestra.run.smithi006.stdout:Paused: No 2024-05-20T22:20:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:01 smithi078 ceph-mon[27225]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:02.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:01 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:02.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:01 smithi006 ceph-mon[24790]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:02.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:02.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:02.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:01 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:02.440 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch ps' 2024-05-20T22:20:03.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:03 smithi078 ceph-mon[27225]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:03.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:03 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:03.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:03 smithi006 ceph-mon[24790]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:03.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:03 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:04.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:04 smithi006 ceph-mon[24790]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c 664f29e9e522 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (5m) 2m ago 5m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (4m) 65s ago 4m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (3m) 2m ago 5m 43.5M - 8.3.5 dad864ee21e9 7461a41cd2f8 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 7ded3806309b 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (4m) 65s ago 4m 411M - 17.2.0 e1d6a67b021e b7780efe93c7 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (6m) 2m ago 7m 45.4M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (4m) 65s ago 4m 38.9M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (5m) 2m ago 5m 19.6M - 1dbe0e931976 b8c7592e4824 2024-05-20T22:20:04.364 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 65s ago 4m 17.2M - 1dbe0e931976 20001161c954 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (4m) 2m ago 4m 34.3M 3217M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (3m) 2m ago 3m 36.9M 3217M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (3m) 2m ago 3m 35.2M 3217M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (2m) 2m ago 2m 27.8M 3217M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (2m) 65s ago 2m 35.1M 3984M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (2m) 65s ago 119s 34.1M 3984M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (95s) 65s ago 95s 30.8M 3984M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (71s) 65s ago 70s 13.3M 3984M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:20:04.365 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (4m) 2m ago 4m 53.2M - 514e6a882f6e 398567f6d4a3 2024-05-20T22:20:04.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:04 smithi078 ceph-mon[27225]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:04.954 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch ls' 2024-05-20T22:20:05.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:05 smithi006 ceph-mon[24790]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:05.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:05 smithi078 ceph-mon[27225]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:crash 2/2 2m ago 6m * 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:mgr 2/2 2m ago 6m count:2 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:mon 2/2 2m ago 5m smithi006:172.21.15.6=smithi006;smithi078:172.21.15.78=smithi078;count:2 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:osd 8 2m ago - 2024-05-20T22:20:06.775 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-05-20T22:20:06.775 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:06 smithi006 ceph-mon[24790]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:06.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:06 smithi078 ceph-mon[27225]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:07.385 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch host ls' 2024-05-20T22:20:08.827 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:08 smithi006 ceph-mon[24790]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:08.828 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:08 smithi006 ceph-mon[24790]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:08.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:08 smithi078 ceph-mon[27225]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:08.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:08 smithi078 ceph-mon[27225]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:09.382 INFO:teuthology.orchestra.run.smithi006.stdout:HOST ADDR LABELS STATUS 2024-05-20T22:20:09.382 INFO:teuthology.orchestra.run.smithi006.stdout:smithi006 172.21.15.6 2024-05-20T22:20:09.382 INFO:teuthology.orchestra.run.smithi006.stdout:smithi078 172.21.15.78 2024-05-20T22:20:09.382 INFO:teuthology.orchestra.run.smithi006.stdout:2 hosts in cluster 2024-05-20T22:20:10.194 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph orch device ls' 2024-05-20T22:20:10.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:10 smithi006 ceph-mon[24790]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:10.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:10 smithi006 ceph-mon[24790]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:10.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:10 smithi078 ceph-mon[27225]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:10.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:10 smithi078 ceph-mon[27225]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:11.699 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:11 smithi006 ceph-mon[24790]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:11.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:11 smithi078 ceph-mon[27225]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:12.105 INFO:teuthology.orchestra.run.smithi006.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-05-20T22:20:12.105 INFO:teuthology.orchestra.run.smithi006.stdout:smithi006 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5330000H400BGN 400G LVM detected, locked 2024-05-20T22:20:12.105 INFO:teuthology.orchestra.run.smithi006.stdout:smithi078 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400QR400BGN 400G LVM detected, locked 2024-05-20T22:20:12.694 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:12 smithi006 ceph-mon[24790]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:12.721 INFO:teuthology.run_tasks:Running task vip.exec... 2024-05-20T22:20:12.753 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:20:12.754 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-05-20T22:20:12.784 INFO:teuthology.orchestra.run.smithi006.stderr:+ systemctl stop nfs-server 2024-05-20T22:20:12.792 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi078.front.sepia.ceph.com 2024-05-20T22:20:12.792 DEBUG:teuthology.orchestra.run.smithi078:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-05-20T22:20:12.822 INFO:teuthology.orchestra.run.smithi078.stderr:+ systemctl stop nfs-server 2024-05-20T22:20:12.831 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T22:20:12.839 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:20:12.839 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-05-20T22:20:12.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:12 smithi078 ceph-mon[27225]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:13.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:13 smithi006 ceph-mon[24790]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:13.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:13 smithi078 ceph-mon[27225]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:16.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:15 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-05-20T22:20:16.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:15 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-05-20T22:20:16.783 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[24786]: 2024-05-20T22:20:16.358+0000 7f42c9376700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:20:17.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: osdmap e60: 8 total, 8 up, 8 in 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: osdmap e61: 8 total, 8 up, 8 in 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: osdmap e62: 8 total, 8 up, 8 in 2024-05-20T22:20:17.041 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: fsmap foofs:0 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: Saving service mds.foofs spec with placement count:2 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:17.042 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:16 smithi006 ceph-mon[24790]: Deploying daemon mds.foofs.smithi006.pburlb on smithi006 2024-05-20T22:20:17.092 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-05-20T22:20:17.103 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-05-20T22:20:17.103 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch ls -f json 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: osdmap e60: 8 total, 8 up, 8 in 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: osdmap e61: 8 total, 8 up, 8 in 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-05-20T22:20:17.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: osdmap e62: 8 total, 8 up, 8 in 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: fsmap foofs:0 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: Saving service mds.foofs spec with placement count:2 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:17.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:16 smithi078 ceph-mon[27225]: Deploying daemon mds.foofs.smithi006.pburlb on smithi006 2024-05-20T22:20:18.415 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:18 smithi006 ceph-mon[24790]: pgmap v211: 65 pgs: 14 creating+peering, 4 active+clean, 47 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:18.415 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:18 smithi006 ceph-mon[24790]: osdmap e63: 8 total, 8 up, 8 in 2024-05-20T22:20:18.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:18 smithi078 ceph-mon[27225]: pgmap v211: 65 pgs: 14 creating+peering, 4 active+clean, 47 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:18.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:18 smithi078 ceph-mon[27225]: osdmap e63: 8 total, 8 up, 8 in 2024-05-20T22:20:19.528 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:19 smithi006 ceph-mon[24790]: osdmap e64: 8 total, 8 up, 8 in 2024-05-20T22:20:19.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:19 smithi078 ceph-mon[27225]: osdmap e64: 8 total, 8 up, 8 in 2024-05-20T22:20:19.967 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:20:19.967 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-20T22:13:47.162244Z", "last_refresh": "2024-05-20T22:17:33.764247Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-20T22:13:40.965658Z", "last_refresh": "2024-05-20T22:17:33.764314Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-20T22:13:43.760647Z", "last_refresh": "2024-05-20T22:17:33.764378Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-20T22:20:16.384167Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-20T22:20:16.380327Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-20T22:13:39.883751Z", "last_refresh": "2024-05-20T22:17:33.764169Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi078:172.21.15.78=smithi078"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-20T22:14:30.115230Z", "last_refresh": "2024-05-20T22:17:33.764022Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-20T22:13:45.827224Z", "last_refresh": "2024-05-20T22:17:33.764442Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-20T22:17:33.764567Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-20T22:13:42.659589Z", "last_refresh": "2024-05-20T22:17:33.764504Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-20T22:20:20.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:20 smithi078 ceph-mon[27225]: pgmap v214: 65 pgs: 14 creating+peering, 9 active+clean, 42 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:20.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:20 smithi006 ceph-mon[24790]: pgmap v214: 65 pgs: 14 creating+peering, 9 active+clean, 42 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:20.808 INFO:tasks.cephadm:mds.foofs has 0/2 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: mds.? [v2:172.21.15.6:6834/3827922570,v1:172.21.15.6:6835/3827922570] up:boot 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: daemon mds.foofs.smithi006.pburlb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: Cluster is now healthy 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: fsmap foofs:0 1 up:standby 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:20:21.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:creating} 2024-05-20T22:20:21.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: daemon mds.foofs.smithi006.pburlb is now active in filesystem foofs as rank 0 2024-05-20T22:20:21.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:21.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:20:21.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T22:20:21.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:21.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:21 smithi078 ceph-mon[27225]: Deploying daemon mds.foofs.smithi078.yocghn on smithi078 2024-05-20T22:20:21.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:21.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: mds.? [v2:172.21.15.6:6834/3827922570,v1:172.21.15.6:6835/3827922570] up:boot 2024-05-20T22:20:21.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: daemon mds.foofs.smithi006.pburlb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: Cluster is now healthy 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: fsmap foofs:0 1 up:standby 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:creating} 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: daemon mds.foofs.smithi006.pburlb is now active in filesystem foofs as rank 0 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:21.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:21 smithi006 ceph-mon[24790]: Deploying daemon mds.foofs.smithi078.yocghn on smithi078 2024-05-20T22:20:21.809 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch ls -f json 2024-05-20T22:20:22.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:22 smithi078 ceph-mon[27225]: pgmap v215: 65 pgs: 14 creating+peering, 42 active+clean, 9 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:22.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:22 smithi078 ceph-mon[27225]: mds.? [v2:172.21.15.6:6834/3827922570,v1:172.21.15.6:6835/3827922570] up:active 2024-05-20T22:20:22.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:22 smithi078 ceph-mon[27225]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 2024-05-20T22:20:22.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:22 smithi006 ceph-mon[24790]: pgmap v215: 65 pgs: 14 creating+peering, 42 active+clean, 9 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-05-20T22:20:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:22 smithi006 ceph-mon[24790]: mds.? [v2:172.21.15.6:6834/3827922570,v1:172.21.15.6:6835/3827922570] up:active 2024-05-20T22:20:22.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:22 smithi006 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 2024-05-20T22:20:23.779 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:20:23.779 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-20T22:13:47.162244Z", "last_refresh": "2024-05-20T22:17:33.764247Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-20T22:13:40.965658Z", "last_refresh": "2024-05-20T22:17:33.764314Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-20T22:13:43.760647Z", "last_refresh": "2024-05-20T22:17:33.764378Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-20T22:20:16.384167Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-20T22:20:16.380327Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-20T22:13:39.883751Z", "last_refresh": "2024-05-20T22:17:33.764169Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi078:172.21.15.78=smithi078"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-20T22:14:30.115230Z", "last_refresh": "2024-05-20T22:17:33.764022Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-20T22:13:45.827224Z", "last_refresh": "2024-05-20T22:17:33.764442Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-20T22:17:33.764567Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-20T22:13:42.659589Z", "last_refresh": "2024-05-20T22:17:33.764504Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-20T22:20:24.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 1 op/s 2024-05-20T22:20:24.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:24.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:24.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:24.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:24.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:boot 2024-05-20T22:20:24.386 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:20:24.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:20:24.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:24 smithi078 ceph-mon[27225]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: pgmap v216: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 1 op/s 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:boot 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:20:24.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:24 smithi006 ceph-mon[24790]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:20:24.541 INFO:tasks.cephadm:mds.foofs has 0/2 2024-05-20T22:20:25.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:25 smithi078 ceph-mon[27225]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:25.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:25.338 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:25.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:25 smithi006 ceph-mon[24790]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:25.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:25.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:25.543 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch ls -f json 2024-05-20T22:20:26.192 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:26 smithi006 ceph-mon[24790]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-05-20T22:20:26.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:26 smithi078 ceph-mon[27225]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-05-20T22:20:28.383 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:20:28.383 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-20T22:13:47.162244Z", "last_refresh": "2024-05-20T22:17:33.764247Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-20T22:13:40.965658Z", "last_refresh": "2024-05-20T22:17:33.764314Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-20T22:13:43.760647Z", "last_refresh": "2024-05-20T22:17:33.764378Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-20T22:20:16.384167Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-20T22:20:16.380327Z", "last_refresh": "2024-05-20T22:20:25.042658Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-20T22:13:39.883751Z", "last_refresh": "2024-05-20T22:17:33.764169Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi078:172.21.15.78=smithi078"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-20T22:14:30.115230Z", "last_refresh": "2024-05-20T22:17:33.764022Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-20T22:13:45.827224Z", "last_refresh": "2024-05-20T22:17:33.764442Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-20T22:17:33.764567Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-20T22:13:42.659589Z", "last_refresh": "2024-05-20T22:17:33.764504Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-20T22:20:28.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:28 smithi006 ceph-mon[24790]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-05-20T22:20:28.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:28 smithi078 ceph-mon[27225]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-05-20T22:20:29.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:29 smithi006 ceph-mon[24790]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:29.222 INFO:tasks.cephadm:mds.foofs has 1/2 2024-05-20T22:20:29.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:29 smithi078 ceph-mon[27225]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:30.223 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch ls -f json 2024-05-20T22:20:30.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:30 smithi078 ceph-mon[27225]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-05-20T22:20:30.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:30 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:30.537 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:30 smithi006 ceph-mon[24790]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-05-20T22:20:30.537 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:30 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:31.465 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:31.466 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:31.466 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:31.466 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:31.466 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:31.466 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:31 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:31.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:31.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:31.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:31.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:31.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:31.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:31 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:32.160 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:20:32.160 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-20T22:13:47.162244Z", "last_refresh": "2024-05-20T22:20:30.162839Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-20T22:13:40.965658Z", "last_refresh": "2024-05-20T22:20:25.042014Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-05-20T22:13:43.760647Z", "last_refresh": "2024-05-20T22:20:30.162992Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-05-20T22:20:16.384167Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-20T22:20:16.380327Z", "last_refresh": "2024-05-20T22:20:25.042658Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-20T22:13:39.883751Z", "last_refresh": "2024-05-20T22:20:25.042166Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi078:172.21.15.78=smithi078"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-20T22:14:30.115230Z", "last_refresh": "2024-05-20T22:20:25.042246Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-20T22:13:45.827224Z", "last_refresh": "2024-05-20T22:20:25.042319Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-05-20T22:20:25.042388Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-20T22:13:42.659589Z", "last_refresh": "2024-05-20T22:20:30.163119Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-20T22:20:32.479 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:32 smithi006 ceph-mon[24790]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-20T22:20:32.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:32 smithi078 ceph-mon[27225]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-20T22:20:32.862 INFO:tasks.cephadm:mds.foofs has 2/2 2024-05-20T22:20:32.862 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T22:20:32.870 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:20:32.871 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-05-20T22:20:33.496 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:33 smithi006 ceph-mon[24790]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:33.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:33 smithi078 ceph-mon[27225]: from='client.14488 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:20:34.425 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:34 smithi006 ceph-mon[24790]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-20T22:20:34.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:34 smithi078 ceph-mon[27225]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-05-20T22:20:35.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:35 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-05-20T22:20:35.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:35 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-05-20T22:20:36.354 INFO:teuthology.orchestra.run.smithi006.stdout:NFS Cluster Created Successfully 2024-05-20T22:20:36.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:36 smithi078 ceph-mon[27225]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:36.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:36 smithi078 ceph-mon[27225]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 2 op/s 2024-05-20T22:20:36.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-05-20T22:20:36.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:36 smithi078 ceph-mon[27225]: osdmap e65: 8 total, 8 up, 8 in 2024-05-20T22:20:36.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:36 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-05-20T22:20:36.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:36 smithi006 ceph-mon[24790]: from='client.14492 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:36.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:36 smithi006 ceph-mon[24790]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 341 B/s wr, 2 op/s 2024-05-20T22:20:36.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-05-20T22:20:36.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:36 smithi006 ceph-mon[24790]: osdmap e65: 8 total, 8 up, 8 in 2024-05-20T22:20:36.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:36 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-05-20T22:20:37.053 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-05-20T22:20:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-05-20T22:20:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: osdmap e66: 8 total, 8 up, 8 in 2024-05-20T22:20:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: Saving service nfs.foo spec with placement count:2 2024-05-20T22:20:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:37.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: Rados config object exists: conf-nfs.foo 2024-05-20T22:20:37.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo-rgw 2024-05-20T22:20:37.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:20:37.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:20:37.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:37.653 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:37 smithi078 ceph-mon[27225]: Deploying daemon nfs.foo.0.0.smithi006.ecmyeo on smithi006 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: osdmap e66: 8 total, 8 up, 8 in 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: Saving service nfs.foo spec with placement count:2 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo 2024-05-20T22:20:37.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: Rados config object exists: conf-nfs.foo 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo-rgw 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:37.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:37 smithi006 ceph-mon[24790]: Deploying daemon nfs.foo.0.0.smithi006.ecmyeo on smithi006 2024-05-20T22:20:38.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:38 smithi006 ceph-mon[24790]: pgmap v225: 97 pgs: 10 creating+peering, 15 unknown, 72 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-05-20T22:20:38.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:38 smithi006 ceph-mon[24790]: osdmap e67: 8 total, 8 up, 8 in 2024-05-20T22:20:38.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:38 smithi006 ceph-mon[24790]: mgrmap e20: smithi006.ssyogp(active, since 5m), standbys: smithi078.vxzcbt 2024-05-20T22:20:38.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:38 smithi078 ceph-mon[27225]: pgmap v225: 97 pgs: 10 creating+peering, 15 unknown, 72 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-05-20T22:20:38.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:38 smithi078 ceph-mon[27225]: osdmap e67: 8 total, 8 up, 8 in 2024-05-20T22:20:38.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:38 smithi078 ceph-mon[27225]: mgrmap e20: smithi006.ssyogp(active, since 5m), standbys: smithi078.vxzcbt 2024-05-20T22:20:39.507 INFO:teuthology.orchestra.run.smithi006.stderr:Invalid command: Unexpected argument '--clusterid' 2024-05-20T22:20:39.507 INFO:teuthology.orchestra.run.smithi006.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-05-20T22:20:39.507 INFO:teuthology.orchestra.run.smithi006.stderr:Error EINVAL: invalid command 2024-05-20T22:20:39.862 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:20:39.863 INFO:teuthology.orchestra.run.smithi006.stdout: "bind": "/fake", 2024-05-20T22:20:39.863 INFO:teuthology.orchestra.run.smithi006.stdout: "fs": "foofs", 2024-05-20T22:20:39.863 INFO:teuthology.orchestra.run.smithi006.stdout: "path": "/", 2024-05-20T22:20:39.863 INFO:teuthology.orchestra.run.smithi006.stdout: "cluster": "foo", 2024-05-20T22:20:39.863 INFO:teuthology.orchestra.run.smithi006.stdout: "mode": "RW" 2024-05-20T22:20:39.863 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:20:40.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:40 smithi006 ceph-mon[24790]: pgmap v227: 97 pgs: 10 creating+peering, 6 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:20:40.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-05-20T22:20:40.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:40 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-05-20T22:20:40.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:40 smithi078 ceph-mon[27225]: pgmap v227: 97 pgs: 10 creating+peering, 6 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:20:40.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-05-20T22:20:40.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:40 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-05-20T22:20:40.653 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-05-20T22:20:41.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:41 smithi006 ceph-mon[24790]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:41.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:41 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:41.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:41 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:20:41.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:41 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-20T22:20:41.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:41 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:20:41.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:41 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:20:41.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:41 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:41.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:41 smithi078 ceph-mon[27225]: from='client.14512 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:41.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:41 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:41.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:41 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:20:41.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:41 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-05-20T22:20:41.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:41 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:20:41.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:41 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:20:41.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:41 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:42.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:42 smithi006 ceph-mon[24790]: pgmap v228: 97 pgs: 10 creating+peering, 87 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:20:42.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:42 smithi006 ceph-mon[24790]: Creating key for client.nfs.foo.1.0.smithi078.metdop 2024-05-20T22:20:42.454 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:42 smithi006 ceph-mon[24790]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-20T22:20:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:42 smithi078 ceph-mon[27225]: pgmap v228: 97 pgs: 10 creating+peering, 87 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:20:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:42 smithi078 ceph-mon[27225]: Creating key for client.nfs.foo.1.0.smithi078.metdop 2024-05-20T22:20:42.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:42 smithi078 ceph-mon[27225]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-20T22:20:44.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:44 smithi006 ceph-mon[24790]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:44.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:44 smithi006 ceph-mon[24790]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-05-20T22:20:44.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:44 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:20:44.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:44 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:20:44.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:44 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:20:44.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:44 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:20:44.541 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:44 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:44.629 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:44 smithi078 ceph-mon[27225]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:44.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:44 smithi078 ceph-mon[27225]: pgmap v229: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-05-20T22:20:44.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:44 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:20:44.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:44 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:20:44.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:44 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:20:44.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:44 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-05-20T22:20:44.630 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:44 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:45 smithi078 ceph-mon[27225]: from='client.14532 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:45 smithi078 ceph-mon[27225]: Rados config object exists: conf-nfs.foo 2024-05-20T22:20:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:45 smithi078 ceph-mon[27225]: Creating key for client.nfs.foo.1.0.smithi078.metdop-rgw 2024-05-20T22:20:45.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:45 smithi078 ceph-mon[27225]: Deploying daemon nfs.foo.1.0.smithi078.metdop on smithi078 2024-05-20T22:20:45.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:45 smithi006 ceph-mon[24790]: from='client.14532 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:45.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:45 smithi006 ceph-mon[24790]: Rados config object exists: conf-nfs.foo 2024-05-20T22:20:45.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:45 smithi006 ceph-mon[24790]: Creating key for client.nfs.foo.1.0.smithi078.metdop-rgw 2024-05-20T22:20:45.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:45 smithi006 ceph-mon[24790]: Deploying daemon nfs.foo.1.0.smithi078.metdop on smithi078 2024-05-20T22:20:46.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:46 smithi078 ceph-mon[27225]: pgmap v230: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-05-20T22:20:46.619 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:46 smithi078 ceph-mon[27225]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:46.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:46 smithi006 ceph-mon[24790]: pgmap v230: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.7 KiB/s wr, 3 op/s 2024-05-20T22:20:46.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:46 smithi006 ceph-mon[24790]: from='client.14546 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:48.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:48.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:48.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:48.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:47 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:48.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:47 smithi006 ceph-mon[24790]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:48.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:47 smithi006 ceph-mon[24790]: pgmap v231: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-05-20T22:20:48.115 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:48.116 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:48.116 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:48.116 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:47 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:48.116 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:47 smithi078 ceph-mon[27225]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:48.116 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:47 smithi078 ceph-mon[27225]: pgmap v231: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-05-20T22:20:49.035 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:48 smithi078 ceph-mon[27225]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:49.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:48 smithi006 ceph-mon[24790]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:50.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:50 smithi078 ceph-mon[27225]: pgmap v232: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-05-20T22:20:50.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:50 smithi078 ceph-mon[27225]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:50.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:50.774 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:50.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:50 smithi006 ceph-mon[24790]: pgmap v232: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-05-20T22:20:50.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:50 smithi006 ceph-mon[24790]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:50.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:50.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.109 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-05-20T22:20:52.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:52.791 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:52 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.828 INFO:teuthology.run_tasks:Running task vip.exec... 2024-05-20T22:20:52.837 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:20:52.838 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: pgmap v233: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:20:52.841 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:20:52.842 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.842 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:52 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:20:52.866 INFO:teuthology.orchestra.run.smithi006.stderr:+ mkdir /mnt/foo 2024-05-20T22:20:52.890 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-05-20T22:20:52.955 INFO:teuthology.orchestra.run.smithi006.stderr:++ hostname 2024-05-20T22:20:52.956 INFO:teuthology.orchestra.run.smithi006.stderr:+ mount -t nfs smithi006:/fake /mnt/foo -o sync 2024-05-20T22:20:53.404 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-05-20T22:20:53.470 INFO:teuthology.orchestra.run.smithi006.stderr:+ echo test 2024-05-20T22:20:53.480 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-05-20T22:20:53.546 INFO:teuthology.orchestra.run.smithi006.stderr:+ sync 2024-05-20T22:20:53.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:53 smithi006 ceph-mon[24790]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:53.830 INFO:teuthology.run_tasks:Running task parallel... 2024-05-20T22:20:53.842 INFO:teuthology.task.parallel:starting parallel... 2024-05-20T22:20:53.842 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-05-20T22:20:53.843 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-05-20T22:20:53.843 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:20:53.843 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-05-20T22:20:53.843 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-05-20T22:20:53.844 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-05-20T22:20:53.844 INFO:teuthology.task.exec:Executing custom commands... 2024-05-20T22:20:53.844 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:20:53.845 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-05-20T22:20:53.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:53 smithi078 ceph-mon[27225]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:20:53.941 INFO:teuthology.orchestra.run.smithi006.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-05-20T22:20:53.941 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:20:53.941 INFO:teuthology.orchestra.run.smithi006.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-05-20T22:20:53.941 INFO:teuthology.orchestra.run.smithi006.stdout:3 of 5 processes prepared for launch 0 sec 2024-05-20T22:20:54.042 INFO:teuthology.orchestra.run.smithi006.stdout:5 of 5 processes prepared for launch 0 sec 2024-05-20T22:20:54.042 INFO:teuthology.orchestra.run.smithi006.stdout:releasing clients 2024-05-20T22:20:54.641 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:54 smithi006 ceph-mon[24790]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-05-20T22:20:54.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:54 smithi078 ceph-mon[27225]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 5 op/s 2024-05-20T22:20:55.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131 28.82 MB/sec warmup 1 sec latency 46.764 ms 2024-05-20T22:20:55.710 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-20T22:20:55.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-05-20T22:20:55.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-05-20T22:20:55.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-05-20T22:20:55.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:20:55.711 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:20:55.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-05-20T22:20:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-05-20T22:20:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-05-20T22:20:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-05-20T22:20:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:20:55.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:20:56.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 290 26.86 MB/sec warmup 2 sec latency 31.024 ms 2024-05-20T22:20:56.747 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:56 smithi006 ceph-mon[24790]: pgmap v235: 97 pgs: 97 active+clean; 2.0 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 142 KiB/s wr, 27 op/s 2024-05-20T22:20:56.747 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:56 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-20T22:20:56.747 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:56 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-05-20T22:20:56.747 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:56 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-05-20T22:20:56.747 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:56 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-05-20T22:20:56.747 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:56 smithi006 ceph-mon[24790]: osdmap e68: 8 total, 8 up, 8 in 2024-05-20T22:20:56.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:56 smithi078 ceph-mon[27225]: pgmap v235: 97 pgs: 97 active+clean; 2.0 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 142 KiB/s wr, 27 op/s 2024-05-20T22:20:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:56 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-05-20T22:20:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:56 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-05-20T22:20:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:56 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-05-20T22:20:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:56 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-05-20T22:20:56.901 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:56 smithi078 ceph-mon[27225]: osdmap e68: 8 total, 8 up, 8 in 2024-05-20T22:20:57.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 432 25.38 MB/sec warmup 3 sec latency 32.086 ms 2024-05-20T22:20:57.390 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-05-20T22:20:57.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:57 smithi006 ceph-mon[24790]: osdmap e69: 8 total, 8 up, 8 in 2024-05-20T22:20:57.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:57 smithi078 ceph-mon[27225]: osdmap e69: 8 total, 8 up, 8 in 2024-05-20T22:20:58.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 573 24.12 MB/sec warmup 4 sec latency 54.673 ms 2024-05-20T22:20:58.747 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:20:58 smithi006 ceph-mon[24790]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 34 MiB data, 65 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 4.2 MiB/s wr, 108 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:20:58.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:20:58 smithi078 ceph-mon[27225]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 34 MiB data, 65 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 4.2 MiB/s wr, 108 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:20:59.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 879 21.65 MB/sec warmup 5 sec latency 54.716 ms 2024-05-20T22:21:00.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 1405 21.68 MB/sec warmup 6 sec latency 24.490 ms 2024-05-20T22:21:00.776 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:00 smithi006 ceph-mon[24790]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 44 MiB data, 70 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 5.4 MiB/s wr, 132 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:00.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:00 smithi078 ceph-mon[27225]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 44 MiB data, 70 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 5.4 MiB/s wr, 132 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:01.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 2379 19.96 MB/sec warmup 7 sec latency 21.500 ms 2024-05-20T22:21:01.701 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph config set global log_to_journald false --force' 2024-05-20T22:21:02.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 3136 21.26 MB/sec warmup 8 sec latency 15.335 ms 2024-05-20T22:21:02.066 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:01 smithi006 ceph-mon[24790]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 84 MiB data, 147 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 11 MiB/s wr, 397 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:02.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:01 smithi078 ceph-mon[27225]: pgmap v240: 97 pgs: 1 peering, 96 active+clean; 84 MiB data, 147 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 11 MiB/s wr, 397 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:03.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 3771 22.00 MB/sec warmup 9 sec latency 34.384 ms 2024-05-20T22:21:04.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 4278 20.18 MB/sec warmup 10 sec latency 22.635 ms 2024-05-20T22:21:04.083 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:04 smithi006 ceph-mon[24790]: pgmap v241: 97 pgs: 97 active+clean; 114 MiB data, 169 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 15 MiB/s wr, 654 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:04.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:04 smithi078 ceph-mon[27225]: pgmap v241: 97 pgs: 97 active+clean; 114 MiB data, 169 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 15 MiB/s wr, 654 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:05.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 5043 20.53 MB/sec warmup 11 sec latency 29.927 ms 2024-05-20T22:21:06.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 5802 19.29 MB/sec warmup 12 sec latency 28.422 ms 2024-05-20T22:21:06.094 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:06 smithi006 ceph-mon[24790]: pgmap v242: 97 pgs: 97 active+clean; 126 MiB data, 212 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 15 MiB/s wr, 817 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:06.402 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:06 smithi078 ceph-mon[27225]: pgmap v242: 97 pgs: 97 active+clean; 126 MiB data, 212 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 15 MiB/s wr, 817 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:06.932 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph mgr module enable nfs --force' 2024-05-20T22:21:07.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 6465 19.97 MB/sec warmup 13 sec latency 49.005 ms 2024-05-20T22:21:08.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 7209 20.70 MB/sec warmup 14 sec latency 19.903 ms 2024-05-20T22:21:08.106 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:08 smithi006 ceph-mon[24790]: pgmap v243: 97 pgs: 97 active+clean; 132 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 15 MiB/s wr, 993 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:08.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:08 smithi078 ceph-mon[27225]: pgmap v243: 97 pgs: 97 active+clean; 132 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 15 MiB/s wr, 993 op/s; 1 B/s, 0 keys/s, 0 objects/s recovering 2024-05-20T22:21:09.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 7598 19.73 MB/sec warmup 15 sec latency 45.705 ms 2024-05-20T22:21:10.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 8153 18.93 MB/sec warmup 16 sec latency 28.746 ms 2024-05-20T22:21:10.124 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:10 smithi006 ceph-mon[24790]: pgmap v244: 97 pgs: 97 active+clean; 133 MiB data, 280 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 13 MiB/s wr, 952 op/s 2024-05-20T22:21:10.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:10 smithi078 ceph-mon[27225]: pgmap v244: 97 pgs: 97 active+clean; 133 MiB data, 280 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 13 MiB/s wr, 952 op/s 2024-05-20T22:21:11.031 INFO:teuthology.orchestra.run.smithi006.stderr:module 'nfs' is already enabled 2024-05-20T22:21:11.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 8710 18.94 MB/sec warmup 17 sec latency 25.771 ms 2024-05-20T22:21:11.128 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:11 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3617130539' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-05-20T22:21:11.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:11 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/3617130539' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-05-20T22:21:12.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 9462 18.34 MB/sec warmup 18 sec latency 31.453 ms 2024-05-20T22:21:12.091 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:12 smithi006 ceph-mon[24790]: pgmap v245: 97 pgs: 97 active+clean; 137 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 13 MiB/s wr, 989 op/s 2024-05-20T22:21:12.092 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:12 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3617130539' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-05-20T22:21:12.092 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:12 smithi006 ceph-mon[24790]: mgrmap e21: smithi006.ssyogp(active, since 6m), standbys: smithi078.vxzcbt 2024-05-20T22:21:12.201 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-05-20T22:21:12.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:12 smithi078 ceph-mon[27225]: pgmap v245: 97 pgs: 97 active+clean; 137 MiB data, 322 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 13 MiB/s wr, 989 op/s 2024-05-20T22:21:12.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:12 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/3617130539' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-05-20T22:21:12.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:12 smithi078 ceph-mon[27225]: mgrmap e21: smithi006.ssyogp(active, since 6m), standbys: smithi078.vxzcbt 2024-05-20T22:21:13.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 10011 18.71 MB/sec warmup 19 sec latency 18.890 ms 2024-05-20T22:21:14.042 INFO:teuthology.orchestra.run.smithi006.stdout: 5 10692 19.10 MB/sec warmup 20 sec latency 96.290 ms 2024-05-20T22:21:14.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:14 smithi006 ceph-mon[24790]: pgmap v246: 97 pgs: 97 active+clean; 140 MiB data, 347 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 11 MiB/s wr, 891 op/s 2024-05-20T22:21:14.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:14 smithi078 ceph-mon[27225]: pgmap v246: 97 pgs: 97 active+clean; 140 MiB data, 347 MiB used, 715 GiB / 715 GiB avail; 8.8 KiB/s rd, 11 MiB/s wr, 891 op/s 2024-05-20T22:21:15.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 11139 18.66 MB/sec warmup 21 sec latency 21.026 ms 2024-05-20T22:21:15.222 INFO:teuthology.orchestra.run.smithi006.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:21:16.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 11725 18.12 MB/sec warmup 22 sec latency 21.723 ms 2024-05-20T22:21:16.090 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-05-20T22:21:16.091 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:21:16.092 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-05-20T22:21:16.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: pgmap v247: 97 pgs: 97 active+clean; 150 MiB data, 380 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 933 op/s 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:16.407 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:16 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: pgmap v247: 97 pgs: 97 active+clean; 150 MiB data, 380 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 933 op/s 2024-05-20T22:21:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:21:16.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:16.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:21:16.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:21:16.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:21:16.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:16.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:16 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:17.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 12346 18.17 MB/sec warmup 23 sec latency 19.175 ms 2024-05-20T22:21:17.411 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:17 smithi006 ceph-mon[24790]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:21:17.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:17 smithi078 ceph-mon[27225]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b 2024-05-20T22:21:18.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 12909 17.62 MB/sec warmup 24 sec latency 43.001 ms 2024-05-20T22:21:18.427 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:18 smithi006 ceph-mon[24790]: pgmap v248: 97 pgs: 97 active+clean; 155 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 875 op/s 2024-05-20T22:21:18.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:18 smithi078 ceph-mon[27225]: pgmap v248: 97 pgs: 97 active+clean; 155 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.6 MiB/s wr, 875 op/s 2024-05-20T22:21:19.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 13285 17.50 MB/sec warmup 25 sec latency 32.211 ms 2024-05-20T22:21:19.570 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:21:20.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 13714 17.59 MB/sec warmup 26 sec latency 31.023 ms 2024-05-20T22:21:20.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:20 smithi006 ceph-mon[24790]: pgmap v249: 97 pgs: 97 active+clean; 155 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 722 op/s 2024-05-20T22:21:20.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:20 smithi006 ceph-mon[24790]: from='client.24409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:20.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:20 smithi078 ceph-mon[27225]: pgmap v249: 97 pgs: 97 active+clean; 155 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.4 MiB/s wr, 722 op/s 2024-05-20T22:21:20.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:20 smithi078 ceph-mon[27225]: from='client.24409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:20.667 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:21:20.667 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (5m) 29s ago 6m 24.9M - ba2b418f427c 664f29e9e522 2024-05-20T22:21:20.667 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (6m) 29s ago 6m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:21:20.667 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (6m) 31s ago 6m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:21:20.667 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (5m) 29s ago 6m 52.8M - 8.3.5 dad864ee21e9 7461a41cd2f8 2024-05-20T22:21:20.667 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (60s) 29s ago 60s 14.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:21:20.667 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (58s) 31s ago 57s 11.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:9283 running (8m) 29s ago 8m 460M - 17.2.0 e1d6a67b021e 7ded3806309b 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (6m) 31s ago 6m 411M - 17.2.0 e1d6a67b021e b7780efe93c7 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 29s ago 8m 47.1M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (6m) 31s ago 6m 36.6M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (39s) 29s ago 39s 48.9M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (34s) 31s ago 33s 50.9M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (6m) 29s ago 6m 21.6M - 1dbe0e931976 b8c7592e4824 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 31s ago 5m 17.9M - 1dbe0e931976 20001161c954 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (5m) 29s ago 5m 43.8M 3217M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (4m) 29s ago 4m 45.4M 3217M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (4m) 29s ago 4m 44.3M 3217M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (4m) 29s ago 4m 42.6M 3217M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (3m) 31s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:21:20.668 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (3m) 31s ago 3m 40.0M 3984M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:21:20.669 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (2m) 31s ago 2m 39.5M 3984M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:21:20.669 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (2m) 31s ago 2m 44.0M 3984M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:21:20.669 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (5m) 29s ago 5m 60.5M - 514e6a882f6e 398567f6d4a3 2024-05-20T22:21:21.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 14147 17.45 MB/sec warmup 27 sec latency 35.236 ms 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:21:21.342 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:21:21.343 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:21:21.464 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:21 smithi006 ceph-mon[24790]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:21.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:21 smithi078 ceph-mon[27225]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:21.946 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:21:21.947 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b", 2024-05-20T22:21:21.947 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:21:21.947 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-05-20T22:21:21.947 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "", 2024-05-20T22:21:21.947 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:8a373da3c102b503bf0a94126977cebf498afc8b image" 2024-05-20T22:21:21.947 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:21:22.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 14473 17.35 MB/sec warmup 28 sec latency 34.526 ms 2024-05-20T22:21:22.469 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:22 smithi006 ceph-mon[24790]: from='client.14606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:22.469 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:22 smithi006 ceph-mon[24790]: pgmap v250: 97 pgs: 97 active+clean; 160 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 722 op/s 2024-05-20T22:21:22.469 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:22 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2718616764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:21:22.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:22 smithi078 ceph-mon[27225]: from='client.14606 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:22.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:22 smithi078 ceph-mon[27225]: pgmap v250: 97 pgs: 97 active+clean; 160 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 7.8 MiB/s wr, 722 op/s 2024-05-20T22:21:22.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:22 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2718616764' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:21:22.655 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:21:23.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 14734 16.82 MB/sec warmup 29 sec latency 40.147 ms 2024-05-20T22:21:23.491 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:23 smithi006 ceph-mon[24790]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:23.491 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:23 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2861007667' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:21:23.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:23 smithi078 ceph-mon[27225]: from='client.14614 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:23.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:23 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2861007667' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:21:24.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 15434 16.77 MB/sec warmup 30 sec latency 23.584 ms 2024-05-20T22:21:24.255 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:24 smithi006 ceph-mon[24790]: pgmap v251: 97 pgs: 97 active+clean; 157 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 733 op/s 2024-05-20T22:21:24.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:24 smithi078 ceph-mon[27225]: pgmap v251: 97 pgs: 97 active+clean; 157 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.3 MiB/s wr, 733 op/s 2024-05-20T22:21:25.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 16189 16.61 MB/sec warmup 31 sec latency 22.022 ms 2024-05-20T22:21:26.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 16653 16.38 MB/sec warmup 32 sec latency 45.173 ms 2024-05-20T22:21:26.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:26 smithi006 ceph-mon[24790]: pgmap v252: 97 pgs: 97 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 733 op/s 2024-05-20T22:21:26.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:26 smithi078 ceph-mon[27225]: pgmap v252: 97 pgs: 97 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 733 op/s 2024-05-20T22:21:27.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 17244 16.70 MB/sec warmup 33 sec latency 36.297 ms 2024-05-20T22:21:28.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 17902 17.03 MB/sec warmup 34 sec latency 25.945 ms 2024-05-20T22:21:28.278 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:28 smithi006 ceph-mon[24790]: pgmap v253: 97 pgs: 97 active+clean; 165 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.7 MiB/s wr, 697 op/s 2024-05-20T22:21:28.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:28 smithi078 ceph-mon[27225]: pgmap v253: 97 pgs: 97 active+clean; 165 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.7 MiB/s wr, 697 op/s 2024-05-20T22:21:29.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 18304 16.66 MB/sec warmup 35 sec latency 65.768 ms 2024-05-20T22:21:30.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 18798 16.41 MB/sec warmup 36 sec latency 24.344 ms 2024-05-20T22:21:30.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:30 smithi006 ceph-mon[24790]: pgmap v254: 97 pgs: 97 active+clean; 172 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.1 MiB/s wr, 689 op/s 2024-05-20T22:21:30.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:30 smithi078 ceph-mon[27225]: pgmap v254: 97 pgs: 97 active+clean; 172 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.1 MiB/s wr, 689 op/s 2024-05-20T22:21:31.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 19501 16.47 MB/sec warmup 37 sec latency 30.272 ms 2024-05-20T22:21:31.822 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:31 smithi006 ceph-mon[24790]: pgmap v255: 97 pgs: 97 active+clean; 173 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 6.7 MiB/s wr, 673 op/s 2024-05-20T22:21:32.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 20271 16.33 MB/sec warmup 38 sec latency 21.946 ms 2024-05-20T22:21:32.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:31 smithi078 ceph-mon[27225]: pgmap v255: 97 pgs: 97 active+clean; 173 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 6.7 MiB/s wr, 673 op/s 2024-05-20T22:21:33.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 21093 16.83 MB/sec warmup 39 sec latency 23.040 ms 2024-05-20T22:21:34.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 21643 16.93 MB/sec warmup 40 sec latency 25.099 ms 2024-05-20T22:21:34.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:34 smithi006 ceph-mon[24790]: pgmap v256: 97 pgs: 97 active+clean; 178 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.6 MiB/s wr, 691 op/s 2024-05-20T22:21:34.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:34 smithi078 ceph-mon[27225]: pgmap v256: 97 pgs: 97 active+clean; 178 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.6 MiB/s wr, 691 op/s 2024-05-20T22:21:35.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 22068 16.60 MB/sec warmup 41 sec latency 19.554 ms 2024-05-20T22:21:36.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 22625 16.62 MB/sec warmup 42 sec latency 21.895 ms 2024-05-20T22:21:36.258 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:36 smithi006 ceph-mon[24790]: pgmap v257: 97 pgs: 97 active+clean; 180 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 751 op/s 2024-05-20T22:21:36.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:36 smithi078 ceph-mon[27225]: pgmap v257: 97 pgs: 97 active+clean; 180 MiB data, 666 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.9 MiB/s wr, 751 op/s 2024-05-20T22:21:37.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 23323 16.43 MB/sec warmup 43 sec latency 24.442 ms 2024-05-20T22:21:38.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 23803 16.29 MB/sec warmup 44 sec latency 43.934 ms 2024-05-20T22:21:38.046 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:38 smithi006 ceph-mon[24790]: pgmap v258: 97 pgs: 97 active+clean; 192 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.2 MiB/s wr, 887 op/s 2024-05-20T22:21:38.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:38 smithi078 ceph-mon[27225]: pgmap v258: 97 pgs: 97 active+clean; 192 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9.2 MiB/s wr, 887 op/s 2024-05-20T22:21:39.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 24319 16.41 MB/sec warmup 45 sec latency 37.699 ms 2024-05-20T22:21:40.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 24912 16.61 MB/sec warmup 46 sec latency 32.303 ms 2024-05-20T22:21:40.086 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:40 smithi006 ceph-mon[24790]: pgmap v259: 97 pgs: 97 active+clean; 199 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 9.1 KiB/s rd, 9.0 MiB/s wr, 892 op/s 2024-05-20T22:21:40.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:40 smithi078 ceph-mon[27225]: pgmap v259: 97 pgs: 97 active+clean; 199 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 9.1 KiB/s rd, 9.0 MiB/s wr, 892 op/s 2024-05-20T22:21:41.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 25239 16.47 MB/sec warmup 47 sec latency 35.789 ms 2024-05-20T22:21:41.849 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:41 smithi006 ceph-mon[24790]: pgmap v260: 97 pgs: 97 active+clean; 195 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.7 MiB/s wr, 774 op/s 2024-05-20T22:21:42.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 25698 16.18 MB/sec warmup 48 sec latency 26.755 ms 2024-05-20T22:21:42.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:41 smithi078 ceph-mon[27225]: pgmap v260: 97 pgs: 97 active+clean; 195 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.7 MiB/s wr, 774 op/s 2024-05-20T22:21:43.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 26136 16.13 MB/sec warmup 49 sec latency 68.008 ms 2024-05-20T22:21:44.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 26889 16.05 MB/sec warmup 50 sec latency 29.073 ms 2024-05-20T22:21:44.161 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:44 smithi006 ceph-mon[24790]: pgmap v261: 97 pgs: 97 active+clean; 194 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 756 op/s 2024-05-20T22:21:44.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:44 smithi078 ceph-mon[27225]: pgmap v261: 97 pgs: 97 active+clean; 194 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 756 op/s 2024-05-20T22:21:45.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 27569 16.13 MB/sec warmup 51 sec latency 40.038 ms 2024-05-20T22:21:46.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 28474 16.55 MB/sec warmup 52 sec latency 22.859 ms 2024-05-20T22:21:46.187 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:46 smithi006 ceph-mon[24790]: pgmap v262: 97 pgs: 97 active+clean; 204 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.4 MiB/s wr, 811 op/s 2024-05-20T22:21:46.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:46 smithi078 ceph-mon[27225]: pgmap v262: 97 pgs: 97 active+clean; 204 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 8.4 MiB/s wr, 811 op/s 2024-05-20T22:21:47.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 28972 16.42 MB/sec warmup 53 sec latency 24.813 ms 2024-05-20T22:21:48.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 29388 16.22 MB/sec warmup 54 sec latency 87.034 ms 2024-05-20T22:21:48.230 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:48 smithi006 ceph-mon[24790]: pgmap v263: 97 pgs: 97 active+clean; 208 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 849 op/s 2024-05-20T22:21:48.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:48 smithi078 ceph-mon[27225]: pgmap v263: 97 pgs: 97 active+clean; 208 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 849 op/s 2024-05-20T22:21:49.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 30102 16.29 MB/sec warmup 55 sec latency 95.512 ms 2024-05-20T22:21:50.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 30840 16.17 MB/sec warmup 56 sec latency 25.451 ms 2024-05-20T22:21:50.149 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:50 smithi006 ceph-mon[24790]: pgmap v264: 97 pgs: 97 active+clean; 212 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.2 MiB/s wr, 764 op/s 2024-05-20T22:21:50.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:50 smithi078 ceph-mon[27225]: pgmap v264: 97 pgs: 97 active+clean; 212 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.2 MiB/s wr, 764 op/s 2024-05-20T22:21:51.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 31393 16.36 MB/sec warmup 57 sec latency 32.139 ms 2024-05-20T22:21:51.999 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:51 smithi006 ceph-mon[24790]: pgmap v265: 97 pgs: 97 active+clean; 216 MiB data, 919 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 702 op/s 2024-05-20T22:21:52.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 32216 16.61 MB/sec warmup 58 sec latency 20.178 ms 2024-05-20T22:21:52.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:51 smithi078 ceph-mon[27225]: pgmap v265: 97 pgs: 97 active+clean; 216 MiB data, 919 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 702 op/s 2024-05-20T22:21:53.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 32536 16.38 MB/sec warmup 59 sec latency 72.343 ms 2024-05-20T22:21:53.223 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:21:54.019 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (5m) 62s ago 7m 24.9M - ba2b418f427c 664f29e9e522 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (7m) 62s ago 7m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (6m) 64s ago 6m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (5m) 62s ago 7m 52.8M - 8.3.5 dad864ee21e9 7461a41cd2f8 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (94s) 62s ago 93s 14.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (91s) 64s ago 90s 11.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:9283 running (8m) 62s ago 8m 460M - 17.2.0 e1d6a67b021e 7ded3806309b 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (6m) 64s ago 6m 411M - 17.2.0 e1d6a67b021e b7780efe93c7 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 62s ago 8m 47.1M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (6m) 64s ago 6m 36.6M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (73s) 62s ago 72s 48.9M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:21:54.020 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (67s) 64s ago 67s 50.9M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 62s ago 7m 21.6M - 1dbe0e931976 b8c7592e4824 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 64s ago 6m 17.9M - 1dbe0e931976 20001161c954 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (5m) 62s ago 5m 43.8M 3217M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (5m) 62s ago 5m 45.4M 3217M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (5m) 62s ago 5m 44.3M 3217M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (4m) 62s ago 4m 42.6M 3217M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (4m) 64s ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (3m) 64s ago 3m 40.0M 3984M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (3m) 64s ago 3m 39.5M 3984M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (3m) 64s ago 3m 44.0M 3984M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:21:54.021 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 62s ago 6m 60.5M - 514e6a882f6e 398567f6d4a3 2024-05-20T22:21:54.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 32867 16.16 MB/sec warmup 60 sec latency 70.066 ms 2024-05-20T22:21:54.255 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:54 smithi006 ceph-mon[24790]: pgmap v266: 97 pgs: 97 active+clean; 219 MiB data, 952 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.0 MiB/s wr, 844 op/s 2024-05-20T22:21:54.255 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:54 smithi006 ceph-mon[24790]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:54.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:54 smithi078 ceph-mon[27225]: pgmap v266: 97 pgs: 97 active+clean; 219 MiB data, 952 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 9.0 MiB/s wr, 844 op/s 2024-05-20T22:21:54.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:54 smithi078 ceph-mon[27225]: from='client.14620 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:54.510 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:21:54.510 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:21:54.510 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:21:54.510 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:54.510 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:21:54.510 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:21:54.511 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:21:55.015 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:21:55.015 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:21:55.015 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:21:55.015 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-05-20T22:21:55.015 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "0/23 daemons upgraded", 2024-05-20T22:21:55.015 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mgr daemons" 2024-05-20T22:21:55.016 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:21:55.044 INFO:teuthology.orchestra.run.smithi006.stdout: 5 33287 16.10 MB/sec warmup 61 sec latency 40.577 ms 2024-05-20T22:21:55.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:55.082 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='client.24425 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:55.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2249702146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:21:55.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:55.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:21:55.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:55.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:21:55.083 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:55 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:21:55.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:55.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='client.24425 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:55.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2249702146' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:21:55.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:55.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:21:55.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:21:55.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:21:55.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:55 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:21:55.606 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:21:56.039 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:56 smithi006 ceph-mon[24790]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (unknown) 2024-05-20T22:21:56.039 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:56 smithi006 ceph-mon[24790]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:21:56.039 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:56 smithi006 ceph-mon[24790]: Upgrade: Need to upgrade myself (mgr.smithi006.ssyogp) 2024-05-20T22:21:56.039 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:56 smithi006 ceph-mon[24790]: pgmap v267: 97 pgs: 97 active+clean; 221 MiB data, 994 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 872 op/s 2024-05-20T22:21:56.039 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:56 smithi006 ceph-mon[24790]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:56 smithi006 ceph-mon[24790]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 on smithi078 2024-05-20T22:21:56.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:56 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2243872915' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:21:56.043 INFO:teuthology.orchestra.run.smithi006.stdout: 5 33964 16.03 MB/sec warmup 62 sec latency 51.032 ms 2024-05-20T22:21:56.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:56 smithi078 ceph-mon[27225]: Upgrade: Target is version 19.0.0-2695-g8a373da3 (unknown) 2024-05-20T22:21:56.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:56 smithi078 ceph-mon[27225]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, digests ['quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72'] 2024-05-20T22:21:56.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:56 smithi078 ceph-mon[27225]: Upgrade: Need to upgrade myself (mgr.smithi006.ssyogp) 2024-05-20T22:21:56.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:56 smithi078 ceph-mon[27225]: pgmap v267: 97 pgs: 97 active+clean; 221 MiB data, 994 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 872 op/s 2024-05-20T22:21:56.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:56 smithi078 ceph-mon[27225]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:21:56.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:56 smithi078 ceph-mon[27225]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 on smithi078 2024-05-20T22:21:56.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:56 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2243872915' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:21:57.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 34541 15.99 MB/sec warmup 63 sec latency 23.358 ms 2024-05-20T22:21:58.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 35364 16.30 MB/sec warmup 64 sec latency 21.525 ms 2024-05-20T22:21:58.061 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:21:58 smithi006 ceph-mon[24790]: pgmap v268: 97 pgs: 97 active+clean; 223 MiB data, 1014 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 823 op/s 2024-05-20T22:21:58.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:21:58 smithi078 ceph-mon[27225]: pgmap v268: 97 pgs: 97 active+clean; 223 MiB data, 1014 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 823 op/s 2024-05-20T22:21:59.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 35868 16.31 MB/sec warmup 65 sec latency 24.327 ms 2024-05-20T22:22:00.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 36475 16.16 MB/sec warmup 66 sec latency 19.048 ms 2024-05-20T22:22:00.071 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:00 smithi006 ceph-mon[24790]: pgmap v269: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 750 op/s 2024-05-20T22:22:00.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:00 smithi078 ceph-mon[27225]: pgmap v269: 97 pgs: 97 active+clean; 225 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.2 MiB/s wr, 750 op/s 2024-05-20T22:22:01.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 37203 16.23 MB/sec warmup 67 sec latency 19.738 ms 2024-05-20T22:22:01.828 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:01 smithi006 ceph-mon[24790]: pgmap v270: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 842 op/s 2024-05-20T22:22:02.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 38043 16.21 MB/sec warmup 68 sec latency 24.213 ms 2024-05-20T22:22:02.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:01 smithi078 ceph-mon[27225]: pgmap v270: 97 pgs: 97 active+clean; 234 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.8 MiB/s wr, 842 op/s 2024-05-20T22:22:03.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 38959 16.53 MB/sec warmup 69 sec latency 11.087 ms 2024-05-20T22:22:04.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 39496 16.52 MB/sec warmup 70 sec latency 30.408 ms 2024-05-20T22:22:04.094 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:04 smithi006 ceph-mon[24790]: pgmap v271: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.6 MiB/s wr, 781 op/s 2024-05-20T22:22:04.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:04 smithi078 ceph-mon[27225]: pgmap v271: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.6 MiB/s wr, 781 op/s 2024-05-20T22:22:05.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 40112 16.42 MB/sec warmup 71 sec latency 31.047 ms 2024-05-20T22:22:06.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 40763 16.43 MB/sec warmup 72 sec latency 48.132 ms 2024-05-20T22:22:06.148 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:06 smithi006 ceph-mon[24790]: pgmap v272: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 812 op/s 2024-05-20T22:22:06.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:06 smithi078 ceph-mon[27225]: pgmap v272: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.9 MiB/s wr, 812 op/s 2024-05-20T22:22:07.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 41703 16.45 MB/sec warmup 73 sec latency 36.011 ms 2024-05-20T22:22:08.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 42728 16.84 MB/sec warmup 74 sec latency 16.419 ms 2024-05-20T22:22:08.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:08 smithi006 ceph-mon[24790]: pgmap v273: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 10 MiB/s wr, 993 op/s 2024-05-20T22:22:08.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:08 smithi078 ceph-mon[27225]: pgmap v273: 97 pgs: 97 active+clean; 254 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 10 MiB/s wr, 993 op/s 2024-05-20T22:22:09.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 43251 16.72 MB/sec warmup 75 sec latency 14.370 ms 2024-05-20T22:22:10.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:10 smithi006 ceph-mon[24790]: pgmap v274: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 10 MiB/s wr, 954 op/s 2024-05-20T22:22:10.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 43815 16.64 MB/sec warmup 76 sec latency 22.673 ms 2024-05-20T22:22:10.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:10 smithi078 ceph-mon[27225]: pgmap v274: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 10 MiB/s wr, 954 op/s 2024-05-20T22:22:11.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 44440 16.63 MB/sec warmup 77 sec latency 32.806 ms 2024-05-20T22:22:12.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 45072 16.56 MB/sec warmup 78 sec latency 22.284 ms 2024-05-20T22:22:12.053 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:11 smithi006 ceph-mon[24790]: pgmap v275: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.8 MiB/s wr, 992 op/s 2024-05-20T22:22:12.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:11 smithi078 ceph-mon[27225]: pgmap v275: 97 pgs: 97 active+clean; 262 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.8 MiB/s wr, 992 op/s 2024-05-20T22:22:13.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 45591 16.67 MB/sec warmup 79 sec latency 40.401 ms 2024-05-20T22:22:14.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 46128 16.69 MB/sec warmup 80 sec latency 38.819 ms 2024-05-20T22:22:14.081 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:14 smithi006 ceph-mon[24790]: pgmap v276: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 922 op/s 2024-05-20T22:22:14.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:14 smithi078 ceph-mon[27225]: pgmap v276: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 922 op/s 2024-05-20T22:22:15.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 46736 16.65 MB/sec warmup 81 sec latency 34.467 ms 2024-05-20T22:22:16.048 INFO:teuthology.orchestra.run.smithi006.stdout: 5 47144 16.50 MB/sec warmup 82 sec latency 31.537 ms 2024-05-20T22:22:16.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:16 smithi006 ceph-mon[24790]: pgmap v277: 97 pgs: 97 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.7 MiB/s wr, 901 op/s 2024-05-20T22:22:16.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:16 smithi078 ceph-mon[27225]: pgmap v277: 97 pgs: 97 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 9.7 MiB/s wr, 901 op/s 2024-05-20T22:22:17.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 47778 16.55 MB/sec warmup 83 sec latency 80.215 ms 2024-05-20T22:22:18.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 48633 16.52 MB/sec warmup 84 sec latency 20.249 ms 2024-05-20T22:22:18.121 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:18 smithi006 ceph-mon[24790]: pgmap v278: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 951 op/s 2024-05-20T22:22:18.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:18 smithi078 ceph-mon[27225]: pgmap v278: 97 pgs: 97 active+clean; 273 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 10 MiB/s wr, 951 op/s 2024-05-20T22:22:19.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 49357 16.69 MB/sec warmup 85 sec latency 46.691 ms 2024-05-20T22:22:20.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 49925 16.74 MB/sec warmup 86 sec latency 28.578 ms 2024-05-20T22:22:20.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:20 smithi078 ceph-mon[27225]: pgmap v279: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 835 op/s 2024-05-20T22:22:20.166 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:20 smithi006 ceph-mon[24790]: pgmap v279: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 835 op/s 2024-05-20T22:22:21.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 50329 16.61 MB/sec warmup 87 sec latency 50.882 ms 2024-05-20T22:22:21.935 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:21 smithi006 ceph-mon[24790]: pgmap v280: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 860 op/s 2024-05-20T22:22:22.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 50858 16.53 MB/sec warmup 88 sec latency 28.168 ms 2024-05-20T22:22:22.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:21 smithi078 ceph-mon[27225]: pgmap v280: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.3 MiB/s wr, 860 op/s 2024-05-20T22:22:23.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 51576 16.54 MB/sec warmup 89 sec latency 36.134 ms 2024-05-20T22:22:24.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 52328 16.52 MB/sec warmup 90 sec latency 24.983 ms 2024-05-20T22:22:24.196 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:24 smithi006 ceph-mon[24790]: pgmap v281: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 796 op/s 2024-05-20T22:22:24.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:24 smithi078 ceph-mon[27225]: pgmap v281: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.9 MiB/s wr, 796 op/s 2024-05-20T22:22:25.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 53188 16.76 MB/sec warmup 91 sec latency 16.321 ms 2024-05-20T22:22:25.956 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:25 smithi006 ceph-mon[24790]: Upgrade: Updating mgr.smithi078.vxzcbt 2024-05-20T22:22:25.957 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:25.957 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:22:25.957 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:22:25.957 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:25 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:22:25.957 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:25 smithi006 ceph-mon[24790]: Deploying daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:22:25.957 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:25 smithi006 ceph-mon[24790]: pgmap v282: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 860 op/s 2024-05-20T22:22:26.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 53702 16.73 MB/sec warmup 92 sec latency 22.151 ms 2024-05-20T22:22:26.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:25 smithi078 ceph-mon[27225]: Upgrade: Updating mgr.smithi078.vxzcbt 2024-05-20T22:22:26.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:26.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:22:26.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:22:26.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:25 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:22:26.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:25 smithi078 ceph-mon[27225]: Deploying daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:22:26.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:25 smithi078 ceph-mon[27225]: pgmap v282: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 860 op/s 2024-05-20T22:22:26.190 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:22:26.963 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:26 smithi006 ceph-mon[24790]: from='client.24433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:27.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 54236 16.63 MB/sec warmup 93 sec latency 32.958 ms 2024-05-20T22:22:27.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:26 smithi078 ceph-mon[27225]: from='client.24433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:27.203 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (6m) 95s ago 8m 24.9M - ba2b418f427c 664f29e9e522 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (8m) 95s ago 8m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (7m) 97s ago 7m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 95s ago 7m 52.8M - 8.3.5 dad864ee21e9 7461a41cd2f8 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (2m) 95s ago 2m 14.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (2m) 97s ago 2m 11.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:9283 running (9m) 95s ago 9m 460M - 17.2.0 e1d6a67b021e 7ded3806309b 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (7m) 97s ago 7m 411M - 17.2.0 e1d6a67b021e b7780efe93c7 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (9m) 95s ago 9m 47.1M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (7m) 97s ago 7m 36.6M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:22:27.204 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (106s) 95s ago 105s 48.9M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (100s) 97s ago 100s 50.9M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 95s ago 7m 21.6M - 1dbe0e931976 b8c7592e4824 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 97s ago 7m 17.9M - 1dbe0e931976 20001161c954 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (6m) 95s ago 6m 43.8M 3217M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (6m) 95s ago 6m 45.4M 3217M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (5m) 95s ago 5m 44.3M 3217M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (5m) 95s ago 5m 42.6M 3217M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (4m) 97s ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (4m) 97s ago 4m 40.0M 3984M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (3m) 97s ago 3m 39.5M 3984M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (3m) 97s ago 3m 44.0M 3984M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:22:27.205 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 95s ago 6m 60.5M - 514e6a882f6e 398567f6d4a3 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:22:27.750 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:22:27.751 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:22:27.751 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:22:27.751 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:22:27.751 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-05-20T22:22:27.751 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:22:27.751 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:22:27.967 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:27 smithi006 ceph-mon[24790]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:27.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:27 smithi006 ceph-mon[24790]: pgmap v283: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-05-20T22:22:27.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:27 smithi006 ceph-mon[24790]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:27.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:27 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3600002609' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:22:28.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 54868 16.62 MB/sec warmup 94 sec latency 30.744 ms 2024-05-20T22:22:28.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:27 smithi078 ceph-mon[27225]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:28.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:27 smithi078 ceph-mon[27225]: pgmap v283: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 1.04k op/s 2024-05-20T22:22:28.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:27 smithi078 ceph-mon[27225]: from='client.14652 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:28.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:27 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/3600002609' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:22:28.252 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:22:28.252 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:22:28.252 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:22:28.252 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-05-20T22:22:28.252 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "0/23 daemons upgraded", 2024-05-20T22:22:28.252 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mgr daemons" 2024-05-20T22:22:28.253 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:22:28.828 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:22:29.026 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:28 smithi006 ceph-mon[24790]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:29.026 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:28 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1955342934' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:22:29.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 55643 16.59 MB/sec warmup 95 sec latency 22.025 ms 2024-05-20T22:22:29.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:28 smithi078 ceph-mon[27225]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:22:29.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:28 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1955342934' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:22:30.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:29 smithi006 ceph-mon[24790]: pgmap v284: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 836 op/s 2024-05-20T22:22:30.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 56228 16.68 MB/sec warmup 96 sec latency 65.410 ms 2024-05-20T22:22:30.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:29 smithi078 ceph-mon[27225]: pgmap v284: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 836 op/s 2024-05-20T22:22:31.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 56766 16.72 MB/sec warmup 97 sec latency 54.178 ms 2024-05-20T22:22:31.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:31 smithi078 ceph-mon[27225]: pgmap v285: 97 pgs: 97 active+clean; 303 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 854 op/s 2024-05-20T22:22:32.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 57155 16.67 MB/sec warmup 98 sec latency 34.443 ms 2024-05-20T22:22:32.073 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:31 smithi006 ceph-mon[24790]: pgmap v285: 97 pgs: 97 active+clean; 303 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 854 op/s 2024-05-20T22:22:33.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 57446 16.53 MB/sec warmup 99 sec latency 44.233 ms 2024-05-20T22:22:34.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 57790 16.43 MB/sec warmup 100 sec latency 56.457 ms 2024-05-20T22:22:34.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:34 smithi006 ceph-mon[24790]: pgmap v286: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 790 op/s 2024-05-20T22:22:34.300 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:34 smithi078 ceph-mon[27225]: pgmap v286: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.0 MiB/s wr, 790 op/s 2024-05-20T22:22:35.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 58276 16.38 MB/sec warmup 101 sec latency 91.773 ms 2024-05-20T22:22:36.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 58925 16.33 MB/sec warmup 102 sec latency 49.649 ms 2024-05-20T22:22:36.180 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:36 smithi006 ceph-mon[24790]: pgmap v287: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.9 MiB/s wr, 832 op/s 2024-05-20T22:22:36.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:36 smithi078 ceph-mon[27225]: pgmap v287: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.9 MiB/s wr, 832 op/s 2024-05-20T22:22:37.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 59432 16.29 MB/sec warmup 103 sec latency 34.911 ms 2024-05-20T22:22:37.438 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:37.439 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:22:37.439 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:22:37.439 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:37 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:22:37.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:37.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:22:37.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:22:37.900 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:37 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:22:38.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 60345 16.51 MB/sec warmup 104 sec latency 18.433 ms 2024-05-20T22:22:38.444 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:38 smithi006 ceph-mon[24790]: pgmap v288: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 824 op/s 2024-05-20T22:22:38.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:38 smithi078 ceph-mon[27225]: pgmap v288: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.7 MiB/s wr, 824 op/s 2024-05-20T22:22:39.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 60825 16.48 MB/sec warmup 105 sec latency 74.049 ms 2024-05-20T22:22:40.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 61353 16.40 MB/sec warmup 106 sec latency 57.143 ms 2024-05-20T22:22:40.455 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:40 smithi006 ceph-mon[24790]: pgmap v289: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 632 op/s 2024-05-20T22:22:40.740 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:40 smithi078 ceph-mon[27225]: pgmap v289: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 632 op/s 2024-05-20T22:22:41.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 61863 16.38 MB/sec warmup 107 sec latency 97.860 ms 2024-05-20T22:22:41.979 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:41 smithi006 ceph-mon[24790]: pgmap v290: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.5 MiB/s wr, 772 op/s 2024-05-20T22:22:42.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 62586 16.32 MB/sec warmup 108 sec latency 58.110 ms 2024-05-20T22:22:42.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:41 smithi078 ceph-mon[27225]: pgmap v290: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.5 MiB/s wr, 772 op/s 2024-05-20T22:22:43.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 63054 16.31 MB/sec warmup 109 sec latency 66.736 ms 2024-05-20T22:22:44.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 63664 16.40 MB/sec warmup 110 sec latency 26.677 ms 2024-05-20T22:22:44.245 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:44 smithi006 ceph-mon[24790]: pgmap v291: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 8.2 MiB/s wr, 781 op/s 2024-05-20T22:22:44.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:44 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:22:44.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:44 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:22:44.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:44 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:22:44.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:44 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:22:44.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:44 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:22:44.246 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:44 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:22:44.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:44 smithi078 ceph-mon[27225]: pgmap v291: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 8.2 MiB/s wr, 781 op/s 2024-05-20T22:22:44.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:44 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:22:44.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:44 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:22:44.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:44 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:22:44.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:44 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:22:44.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:44 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:22:44.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:44 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:22:45.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 64192 16.44 MB/sec warmup 111 sec latency 31.807 ms 2024-05-20T22:22:45.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:45 smithi006 ceph-mon[24790]: mgrmap e22: smithi006.ssyogp(active, since 7m), standbys: smithi078.vxzcbt 2024-05-20T22:22:45.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:45 smithi078 ceph-mon[27225]: mgrmap e22: smithi006.ssyogp(active, since 7m), standbys: smithi078.vxzcbt 2024-05-20T22:22:46.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 64655 16.33 MB/sec warmup 112 sec latency 39.194 ms 2024-05-20T22:22:46.260 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:46 smithi006 ceph-mon[24790]: pgmap v292: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.8 MiB/s wr, 790 op/s 2024-05-20T22:22:46.260 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:46 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:46.260 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:46 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:46.332 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:46 smithi078 ceph-mon[27225]: pgmap v292: 97 pgs: 97 active+clean; 322 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.8 MiB/s wr, 790 op/s 2024-05-20T22:22:46.332 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:46 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:46.332 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:46 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:47.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 65190 16.28 MB/sec warmup 113 sec latency 22.408 ms 2024-05-20T22:22:48.049 INFO:teuthology.orchestra.run.smithi006.stdout: 5 65862 16.27 MB/sec warmup 114 sec latency 47.473 ms 2024-05-20T22:22:48.274 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:48 smithi006 ceph-mon[24790]: pgmap v293: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.8 MiB/s wr, 948 op/s 2024-05-20T22:22:48.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:48 smithi078 ceph-mon[27225]: pgmap v293: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 8.8 MiB/s wr, 948 op/s 2024-05-20T22:22:49.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 66411 16.24 MB/sec warmup 115 sec latency 27.252 ms 2024-05-20T22:22:50.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 67001 16.32 MB/sec warmup 116 sec latency 24.517 ms 2024-05-20T22:22:50.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: pgmap v294: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.2 MiB/s wr, 786 op/s 2024-05-20T22:22:50.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:50.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:50.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:50.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:22:50.614 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: Upgrade: Need to upgrade myself (mgr.smithi006.ssyogp) 2024-05-20T22:22:50.614 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: Failing over to other MGR 2024-05-20T22:22:50.614 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr fail", "who": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:22:50.614 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:50 smithi006 ceph-mon[24790]: osdmap e70: 8 total, 8 up, 8 in 2024-05-20T22:22:50.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: pgmap v294: 97 pgs: 97 active+clean; 333 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 8.2 MiB/s wr, 786 op/s 2024-05-20T22:22:50.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:50.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:50.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' 2024-05-20T22:22:50.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:22:50.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: Upgrade: Need to upgrade myself (mgr.smithi006.ssyogp) 2024-05-20T22:22:50.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: Failing over to other MGR 2024-05-20T22:22:50.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr fail", "who": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:22:50.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:50 smithi078 ceph-mon[27225]: osdmap e70: 8 total, 8 up, 8 in 2024-05-20T22:22:51.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 67423 16.31 MB/sec warmup 117 sec latency 48.905 ms 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "mgr fail", "who": "smithi006.ssyogp"}]': finished 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: mgrmap e23: smithi078.vxzcbt(active, starting, since 0.996975s) 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:22:51.625 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: Manager daemon smithi078.vxzcbt is now available 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:22:51.626 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/trash_purge_schedule"}]: dispatch 2024-05-20T22:22:51.627 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/trash_purge_schedule"}]: dispatch 2024-05-20T22:22:51.627 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: Deploying cephadm binary to smithi078 2024-05-20T22:22:51.627 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: Deploying cephadm binary to smithi006 2024-05-20T22:22:51.627 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:51 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.14206 172.21.15.6:0/264088059' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "mgr fail", "who": "smithi006.ssyogp"}]': finished 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: mgrmap e23: smithi078.vxzcbt(active, starting, since 0.996975s) 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:22:51.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: Manager daemon smithi078.vxzcbt is now available 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/trash_purge_schedule"}]: dispatch 2024-05-20T22:22:51.651 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi078.vxzcbt/trash_purge_schedule"}]: dispatch 2024-05-20T22:22:51.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: Deploying cephadm binary to smithi078 2024-05-20T22:22:51.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: Deploying cephadm binary to smithi006 2024-05-20T22:22:51.652 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:51 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:22:52.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 67855 16.28 MB/sec warmup 118 sec latency 36.349 ms 2024-05-20T22:22:52.628 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:52 smithi006 ceph-mon[24790]: mgrmap e24: smithi078.vxzcbt(active, since 2s) 2024-05-20T22:22:52.628 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:52 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:22:52.628 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:52 smithi006 ceph-mon[24790]: pgmap v3: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:52.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:52 smithi078 ceph-mon[27225]: mgrmap e24: smithi078.vxzcbt(active, since 2s) 2024-05-20T22:22:52.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:52 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:22:52.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:52 smithi078 ceph-mon[27225]: pgmap v3: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:53.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 68342 16.18 MB/sec warmup 119 sec latency 27.246 ms 2024-05-20T22:22:53.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:53 smithi006 ceph-mon[24790]: pgmap v4: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:53.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:53 smithi006 ceph-mon[24790]: mgrmap e25: smithi078.vxzcbt(active, since 3s) 2024-05-20T22:22:53.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:53 smithi006 ceph-mon[24790]: [20/May/2024:22:22:52] ENGINE Bus STARTING 2024-05-20T22:22:53.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:53 smithi006 ceph-mon[24790]: [20/May/2024:22:22:52] ENGINE Serving on https://172.21.15.78:7150 2024-05-20T22:22:53.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:53 smithi006 ceph-mon[24790]: [20/May/2024:22:22:52] ENGINE Serving on http://172.21.15.78:8765 2024-05-20T22:22:53.432 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:53 smithi006 ceph-mon[24790]: [20/May/2024:22:22:52] ENGINE Bus STARTED 2024-05-20T22:22:53.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:53 smithi078 ceph-mon[27225]: pgmap v4: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:53.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:53 smithi078 ceph-mon[27225]: mgrmap e25: smithi078.vxzcbt(active, since 3s) 2024-05-20T22:22:53.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:53 smithi078 ceph-mon[27225]: [20/May/2024:22:22:52] ENGINE Bus STARTING 2024-05-20T22:22:53.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:53 smithi078 ceph-mon[27225]: [20/May/2024:22:22:52] ENGINE Serving on https://172.21.15.78:7150 2024-05-20T22:22:53.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:53 smithi078 ceph-mon[27225]: [20/May/2024:22:22:52] ENGINE Serving on http://172.21.15.78:8765 2024-05-20T22:22:53.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:53 smithi078 ceph-mon[27225]: [20/May/2024:22:22:52] ENGINE Bus STARTED 2024-05-20T22:22:55.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 69623 8.19 MB/sec execute 1 sec latency 46.160 ms 2024-05-20T22:22:55.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:55 smithi006 ceph-mon[24790]: pgmap v5: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:55.646 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:55 smithi006 ceph-mon[24790]: mgrmap e26: smithi078.vxzcbt(active, since 5s) 2024-05-20T22:22:55.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:55 smithi078 ceph-mon[27225]: pgmap v5: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:55.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:55 smithi078 ceph-mon[27225]: mgrmap e26: smithi078.vxzcbt(active, since 5s) 2024-05-20T22:22:56.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 70233 14.83 MB/sec execute 2 sec latency 27.778 ms 2024-05-20T22:22:57.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 70756 17.02 MB/sec execute 3 sec latency 41.842 ms 2024-05-20T22:22:57.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:57 smithi078 ceph-mon[27225]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:57.914 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:57 smithi006 ceph-mon[24790]: pgmap v6: 97 pgs: 97 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:22:58.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 71194 17.43 MB/sec execute 4 sec latency 31.755 ms 2024-05-20T22:22:59.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 71525 14.69 MB/sec execute 5 sec latency 31.989 ms 2024-05-20T22:22:59.359 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:22:59.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:22:59 smithi006 ceph-mon[24790]: pgmap v7: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-05-20T22:23:00.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 71891 12.88 MB/sec execute 6 sec latency 32.614 ms 2024-05-20T22:23:00.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:22:59 smithi078 ceph-mon[27225]: pgmap v7: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 5.6 MiB/s wr, 587 op/s 2024-05-20T22:23:00.348 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:23:00.348 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (7m) 2m ago 8m 24.9M - ba2b418f427c 664f29e9e522 2024-05-20T22:23:00.348 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (8m) 2m ago 8m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:23:00.348 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (7m) 15s ago 7m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:23:00.348 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 2m ago 8m 52.8M - 8.3.5 dad864ee21e9 7461a41cd2f8 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (2m) 2m ago 2m 14.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (2m) 15s ago 2m 11.5M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e 7ded3806309b 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (28s) 15s ago 7m 113M - 19.0.0-2695-g8a373da3 86045c3f4e75 ab4289c5f48c 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (9m) 2m ago 10m 47.1M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (7m) 15s ago 7m 33.8M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (2m) 2m ago 2m 48.9M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (2m) 15s ago 2m 75.2M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (8m) 2m ago 8m 21.6M - 1dbe0e931976 b8c7592e4824 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 15s ago 7m 17.6M - 1dbe0e931976 20001161c954 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (7m) 2m ago 7m 43.8M 3217M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (6m) 2m ago 6m 45.4M 3217M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (6m) 2m ago 6m 44.3M 3217M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:23:00.349 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (5m) 2m ago 5m 42.6M 3217M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:23:00.350 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (5m) 15s ago 5m 237M 3984M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:23:00.350 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (4m) 15s ago 4m 247M 3984M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:23:00.350 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (4m) 15s ago 4m 234M 3984M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:23:00.350 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (4m) 15s ago 4m 348M 3984M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:23:00.350 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 2m ago 7m 60.5M - 514e6a882f6e 398567f6d4a3 2024-05-20T22:23:00.921 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:23:00.921 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:23:00.921 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:23:00.921 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:23:00.922 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:23:00.923 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:23:00.923 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:23:00.923 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:23:00.967 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:00.967 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:00.967 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: Standby manager daemon smithi006.ssyogp started 2024-05-20T22:23:00.967 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/crt"}]: dispatch 2024-05-20T22:23:00.967 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:23:00.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/key"}]: dispatch 2024-05-20T22:23:00.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:23:00.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:00.968 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:00 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:01.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 72255 12.30 MB/sec execute 7 sec latency 47.412 ms 2024-05-20T22:23:01.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:01.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:01.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: Standby manager daemon smithi006.ssyogp started 2024-05-20T22:23:01.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/crt"}]: dispatch 2024-05-20T22:23:01.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:23:01.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/key"}]: dispatch 2024-05-20T22:23:01.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/2530238698' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:23:01.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:01.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:00 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:01.366 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "1/23 daemons upgraded", 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:23:01.367 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:23:01.933 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:23:01.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:01 smithi006 ceph-mon[24790]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:01.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:01 smithi006 ceph-mon[24790]: pgmap v8: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.4 MiB/s wr, 456 op/s 2024-05-20T22:23:01.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:01 smithi006 ceph-mon[24790]: mgrmap e27: smithi078.vxzcbt(active, since 11s), standbys: smithi006.ssyogp 2024-05-20T22:23:01.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:01 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:23:01.979 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:01 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/3164875930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:23:02.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 72732 12.57 MB/sec execute 8 sec latency 48.154 ms 2024-05-20T22:23:02.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:01 smithi078 ceph-mon[27225]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:02.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:01 smithi078 ceph-mon[27225]: pgmap v8: 97 pgs: 97 active+clean; 319 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 4.4 MiB/s wr, 456 op/s 2024-05-20T22:23:02.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:01 smithi078 ceph-mon[27225]: mgrmap e27: smithi078.vxzcbt(active, since 11s), standbys: smithi006.ssyogp 2024-05-20T22:23:02.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:01 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:23:02.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:01 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/3164875930' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:23:02.982 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:02 smithi006 ceph-mon[24790]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:02.982 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:02 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1505886624' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:23:02.982 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:02 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:02.982 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:02 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:03.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:02 smithi078 ceph-mon[27225]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:03.029 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:02 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1505886624' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:23:03.030 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:02 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:03.030 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:02 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:03.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 73484 12.53 MB/sec execute 9 sec latency 27.742 ms 2024-05-20T22:23:04.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 74219 14.35 MB/sec execute 10 sec latency 39.763 ms 2024-05-20T22:23:04.238 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:04 smithi006 ceph-mon[24790]: pgmap v9: 97 pgs: 97 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.4 MiB/s wr, 647 op/s 2024-05-20T22:23:04.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:04 smithi078 ceph-mon[27225]: pgmap v9: 97 pgs: 97 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 6.4 MiB/s wr, 647 op/s 2024-05-20T22:23:05.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 74678 14.63 MB/sec execute 11 sec latency 37.678 ms 2024-05-20T22:23:06.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 74962 14.04 MB/sec execute 12 sec latency 74.127 ms 2024-05-20T22:23:06.258 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:06 smithi006 ceph-mon[24790]: pgmap v10: 97 pgs: 97 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.9 MiB/s wr, 592 op/s 2024-05-20T22:23:06.258 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:06 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:06.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:06 smithi078 ceph-mon[27225]: pgmap v10: 97 pgs: 97 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.9 MiB/s wr, 592 op/s 2024-05-20T22:23:06.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:06 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:07.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 75317 13.21 MB/sec execute 13 sec latency 39.186 ms 2024-05-20T22:23:07.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: pgmap v11: 97 pgs: 97 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.9 MiB/s wr, 592 op/s 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi078 to 2704M 2024-05-20T22:23:07.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:07 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: pgmap v11: 97 pgs: 97 active+clean; 317 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.9 MiB/s wr, 592 op/s 2024-05-20T22:23:07.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:07.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi078 to 2704M 2024-05-20T22:23:07.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:07 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:08.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 75936 13.18 MB/sec execute 14 sec latency 24.131 ms 2024-05-20T22:23:09.023 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:09.023 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:09.023 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.023 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.023 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: Adjusting osd_memory_target on smithi006 to 1937M 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:23:09.024 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:09 smithi006 ceph-mon[24790]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:23:09.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 76660 13.27 MB/sec execute 15 sec latency 33.768 ms 2024-05-20T22:23:09.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:09.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:09.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: Adjusting osd_memory_target on smithi006 to 1937M 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:23:09.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:23:09.401 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:09 smithi078 ceph-mon[27225]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:23:10.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 77162 13.29 MB/sec execute 16 sec latency 26.407 ms 2024-05-20T22:23:10.311 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 9.2 MiB/s wr, 891 op/s 2024-05-20T22:23:10.311 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:23:10.311 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:23:10.311 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 9.2 MiB/s wr, 891 op/s 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:11.050 INFO:teuthology.orchestra.run.smithi006.stdout: 5 77818 14.10 MB/sec execute 17 sec latency 35.317 ms 2024-05-20T22:23:11.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:11 smithi006 ceph-mon[24790]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-05-20T22:23:11.401 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:11 smithi006 ceph-mon[24790]: Reconfiguring daemon alertmanager.smithi006 on smithi006 2024-05-20T22:23:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:11 smithi078 ceph-mon[27225]: Reconfiguring alertmanager.smithi006 (dependencies changed)... 2024-05-20T22:23:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:11 smithi078 ceph-mon[27225]: Reconfiguring daemon alertmanager.smithi006 on smithi006 2024-05-20T22:23:12.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 78337 14.51 MB/sec execute 18 sec latency 43.411 ms 2024-05-20T22:23:12.415 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:12 smithi006 ceph-mon[24790]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 550 op/s 2024-05-20T22:23:12.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:12 smithi078 ceph-mon[27225]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 550 op/s 2024-05-20T22:23:13.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 78712 14.00 MB/sec execute 19 sec latency 38.880 ms 2024-05-20T22:23:14.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 79156 13.64 MB/sec execute 20 sec latency 29.783 ms 2024-05-20T22:23:14.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:14 smithi006 ceph-mon[24790]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 889 op/s 2024-05-20T22:23:14.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:14 smithi078 ceph-mon[27225]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 889 op/s 2024-05-20T22:23:15.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 79588 13.56 MB/sec execute 21 sec latency 60.149 ms 2024-05-20T22:23:16.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 80246 13.54 MB/sec execute 22 sec latency 43.487 ms 2024-05-20T22:23:16.239 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:16 smithi006 ceph-mon[24790]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-05-20T22:23:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:16 smithi078 ceph-mon[27225]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-05-20T22:23:17.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 80721 13.43 MB/sec execute 23 sec latency 41.018 ms 2024-05-20T22:23:18.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 81317 13.91 MB/sec execute 24 sec latency 39.539 ms 2024-05-20T22:23:18.256 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:18 smithi006 ceph-mon[24790]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-05-20T22:23:18.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:18 smithi078 ceph-mon[27225]: pgmap v16: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.8 MiB/s wr, 638 op/s 2024-05-20T22:23:19.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 81791 14.05 MB/sec execute 25 sec latency 43.435 ms 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: pgmap v17: 97 pgs: 97 active+clean; 312 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 935 op/s 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:23:19.773 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:23:19.774 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:19 smithi006 ceph-mon[24790]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: pgmap v17: 97 pgs: 97 active+clean; 312 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.5 MiB/s wr, 935 op/s 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: Reconfiguring grafana.smithi006 (dependencies changed)... 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: Regenerating cephadm self-signed grafana TLS certificates 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-05-20T22:23:19.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:19 smithi078 ceph-mon[27225]: Reconfiguring daemon grafana.smithi006 on smithi006 2024-05-20T22:23:20.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 82132 13.88 MB/sec execute 26 sec latency 28.878 ms 2024-05-20T22:23:20.783 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:20 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:20.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:20 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:21.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 82624 13.60 MB/sec execute 27 sec latency 47.320 ms 2024-05-20T22:23:22.038 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:21 smithi006 ceph-mon[24790]: pgmap v18: 97 pgs: 97 active+clean; 312 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 636 op/s 2024-05-20T22:23:22.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 83191 13.71 MB/sec execute 28 sec latency 31.609 ms 2024-05-20T22:23:22.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:21 smithi078 ceph-mon[27225]: pgmap v18: 97 pgs: 97 active+clean; 312 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 636 op/s 2024-05-20T22:23:23.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 83823 13.61 MB/sec execute 29 sec latency 31.917 ms 2024-05-20T22:23:24.048 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:23 smithi006 ceph-mon[24790]: pgmap v19: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 925 op/s 2024-05-20T22:23:24.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 84343 13.60 MB/sec execute 30 sec latency 31.242 ms 2024-05-20T22:23:24.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:23 smithi078 ceph-mon[27225]: pgmap v19: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 925 op/s 2024-05-20T22:23:25.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 84804 13.73 MB/sec execute 31 sec latency 40.746 ms 2024-05-20T22:23:25.848 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:25 smithi006 ceph-mon[24790]: pgmap v20: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 586 op/s 2024-05-20T22:23:26.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 85359 13.97 MB/sec execute 32 sec latency 24.773 ms 2024-05-20T22:23:26.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:25 smithi078 ceph-mon[27225]: pgmap v20: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 586 op/s 2024-05-20T22:23:27.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 85653 13.84 MB/sec execute 33 sec latency 30.346 ms 2024-05-20T22:23:27.617 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:27 smithi006 ceph-mon[24790]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 586 op/s 2024-05-20T22:23:27.617 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:27 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:27.617 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:27 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:27.617 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:27 smithi006 ceph-mon[24790]: Reconfiguring node-exporter.smithi006 (dependencies changed)... 2024-05-20T22:23:27.617 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:27 smithi006 ceph-mon[24790]: Reconfiguring daemon node-exporter.smithi006 on smithi006 2024-05-20T22:23:27.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:27 smithi078 ceph-mon[27225]: pgmap v21: 97 pgs: 97 active+clean; 322 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.9 MiB/s wr, 586 op/s 2024-05-20T22:23:27.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:27 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:27.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:27 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:27.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:27 smithi078 ceph-mon[27225]: Reconfiguring node-exporter.smithi006 (dependencies changed)... 2024-05-20T22:23:27.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:27 smithi078 ceph-mon[27225]: Reconfiguring daemon node-exporter.smithi006 on smithi006 2024-05-20T22:23:28.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 86065 13.58 MB/sec execute 34 sec latency 63.214 ms 2024-05-20T22:23:29.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 86523 13.41 MB/sec execute 35 sec latency 38.504 ms 2024-05-20T22:23:29.654 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:29 smithi006 ceph-mon[24790]: pgmap v22: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 895 op/s 2024-05-20T22:23:29.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:29 smithi078 ceph-mon[27225]: pgmap v22: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.3 MiB/s wr, 895 op/s 2024-05-20T22:23:30.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 86961 13.45 MB/sec execute 36 sec latency 35.023 ms 2024-05-20T22:23:31.052 INFO:teuthology.orchestra.run.smithi006.stdout: 5 87562 13.33 MB/sec execute 37 sec latency 53.949 ms 2024-05-20T22:23:31.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:31 smithi006 ceph-mon[24790]: pgmap v23: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.6 MiB/s wr, 597 op/s 2024-05-20T22:23:31.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:31 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:31.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:31 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:31.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:31 smithi006 ceph-mon[24790]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-05-20T22:23:31.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:31 smithi006 ceph-mon[24790]: Reconfiguring daemon prometheus.smithi006 on smithi006 2024-05-20T22:23:32.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 87910 13.26 MB/sec execute 38 sec latency 32.381 ms 2024-05-20T22:23:32.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:31 smithi078 ceph-mon[27225]: pgmap v23: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.6 MiB/s wr, 597 op/s 2024-05-20T22:23:32.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:31 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:32.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:31 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:32.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:31 smithi078 ceph-mon[27225]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-05-20T22:23:32.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:31 smithi078 ceph-mon[27225]: Reconfiguring daemon prometheus.smithi006 on smithi006 2024-05-20T22:23:32.393 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:23:33.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 88317 13.36 MB/sec execute 39 sec latency 29.137 ms 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 starting - - - - 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (9m) 26s ago 9m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (8m) 32s ago 8m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 starting - - - - 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (3m) 26s ago 3m 35.9M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (3m) 32s ago 3m 11.8M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:9283 running (10m) 26s ago 10m 316M - 17.2.0 e1d6a67b021e 7ded3806309b 2024-05-20T22:23:33.278 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (61s) 32s ago 8m 491M - 19.0.0-2695-g8a373da3 86045c3f4e75 ab4289c5f48c 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (10m) 26s ago 10m 51.8M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (8m) 32s ago 8m 36.7M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (2m) 26s ago 2m 256M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (2m) 32s ago 2m 75.4M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 starting - - - - 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 32s ago 8m 20.7M - 1.3.1 1dbe0e931976 20001161c954 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (7m) 26s ago 7m 293M 1937M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (7m) 26s ago 7m 234M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (6m) 26s ago 6m 178M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (6m) 26s ago 6m 255M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (5m) 32s ago 5m 269M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (5m) 32s ago 5m 296M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (5m) 32s ago 5m 266M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:23:33.279 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (4m) 32s ago 4m 384M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:23:33.280 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 26s ago 7m 63.8M - 2.33.4 514e6a882f6e 398567f6d4a3 2024-05-20T22:23:33.725 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:33 smithi006 ceph-mon[24790]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:33.725 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:33 smithi006 ceph-mon[24790]: pgmap v24: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 869 op/s 2024-05-20T22:23:33.725 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:33 smithi006 ceph-mon[24790]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:33.725 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:33 smithi006 ceph-mon[24790]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:33.851 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:23:33.851 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:23:33.851 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:23:33.851 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:33.851 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:23:33.851 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T22:23:33.851 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:23:33.852 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:23:34.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 88840 13.53 MB/sec execute 40 sec latency 35.420 ms 2024-05-20T22:23:34.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:33 smithi078 ceph-mon[27225]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:34.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:33 smithi078 ceph-mon[27225]: pgmap v24: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.9 MiB/s wr, 869 op/s 2024-05-20T22:23:34.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:33 smithi078 ceph-mon[27225]: from='client.24505 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:34.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:33 smithi078 ceph-mon[27225]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:34.390 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:23:34.390 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:23:34.390 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:23:34.390 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:23:34.391 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-05-20T22:23:34.391 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "1/23 daemons upgraded", 2024-05-20T22:23:34.391 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-05-20T22:23:34.391 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:23:34.391 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:23:34.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:34 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/1564397583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:23:35.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 89234 13.48 MB/sec execute 41 sec latency 36.816 ms 2024-05-20T22:23:35.114 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:23:35.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:34 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/1564397583' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:23:35.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:35 smithi006 ceph-mon[24790]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:35.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:35 smithi006 ceph-mon[24790]: pgmap v25: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 580 op/s 2024-05-20T22:23:35.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:35 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/4175023955' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:23:35.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:35 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:36.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 89786 13.30 MB/sec execute 42 sec latency 37.708 ms 2024-05-20T22:23:36.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:35 smithi078 ceph-mon[27225]: from='client.14734 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:23:36.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:35 smithi078 ceph-mon[27225]: pgmap v25: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 580 op/s 2024-05-20T22:23:36.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:35 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/4175023955' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:23:36.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:35 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:37.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 90321 13.39 MB/sec execute 43 sec latency 26.383 ms 2024-05-20T22:23:37.932 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:37 smithi006 ceph-mon[24790]: pgmap v26: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 580 op/s 2024-05-20T22:23:38.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 91037 13.36 MB/sec execute 44 sec latency 30.701 ms 2024-05-20T22:23:38.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:37 smithi078 ceph-mon[27225]: pgmap v26: 97 pgs: 97 active+clean; 310 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 580 op/s 2024-05-20T22:23:39.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 91809 13.68 MB/sec execute 45 sec latency 21.688 ms 2024-05-20T22:23:39.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:39 smithi006 ceph-mon[24790]: pgmap v27: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 856 op/s 2024-05-20T22:23:39.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:39 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:39.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:39 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:40.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 92529 13.95 MB/sec execute 46 sec latency 38.910 ms 2024-05-20T22:23:40.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:39 smithi078 ceph-mon[27225]: pgmap v27: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 856 op/s 2024-05-20T22:23:40.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:39 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:40.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:39 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:40.954 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:40 smithi006 ceph-mon[24790]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-05-20T22:23:40.954 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:40 smithi006 ceph-mon[24790]: Reconfiguring daemon node-exporter.smithi078 on smithi078 2024-05-20T22:23:41.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 93001 13.86 MB/sec execute 47 sec latency 37.366 ms 2024-05-20T22:23:41.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:40 smithi078 ceph-mon[27225]: Reconfiguring node-exporter.smithi078 (dependencies changed)... 2024-05-20T22:23:41.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:40 smithi078 ceph-mon[27225]: Reconfiguring daemon node-exporter.smithi078 on smithi078 2024-05-20T22:23:41.958 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:41 smithi006 ceph-mon[24790]: pgmap v28: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 547 op/s 2024-05-20T22:23:41.958 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:41 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:41.958 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:41 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:41.996 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:41 smithi078 ceph-mon[27225]: pgmap v28: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 547 op/s 2024-05-20T22:23:41.996 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:41 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:41.996 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:41 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:42.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 93406 13.65 MB/sec execute 48 sec latency 49.679 ms 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:23:42.993 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:23:42.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:23:42.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:42.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:23:42.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:23:42.994 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:42 smithi006 ceph-mon[24790]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:23:43.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 93709 13.49 MB/sec execute 49 sec latency 36.446 ms 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi006.front.sepia.ceph.com:9093"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi006.front.sepia.ceph.com:3000"}]: dispatch 2024-05-20T22:23:43.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi006.front.sepia.ceph.com:9095"}]: dispatch 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:23:43.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:42 smithi078 ceph-mon[27225]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:23:44.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:43 smithi006 ceph-mon[24790]: pgmap v29: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2024-05-20T22:23:44.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 94215 13.55 MB/sec execute 50 sec latency 47.469 ms 2024-05-20T22:23:44.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:43 smithi078 ceph-mon[27225]: pgmap v29: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.3 MiB/s wr, 913 op/s 2024-05-20T22:23:45.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 94788 13.50 MB/sec execute 51 sec latency 31.485 ms 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: Upgrade: Updating mgr.smithi006.ssyogp 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: pgmap v30: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.0 MiB/s wr, 641 op/s 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:23:45.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:45 smithi006 ceph-mon[24790]: Deploying daemon mgr.smithi006.ssyogp on smithi006 2024-05-20T22:23:45.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: Upgrade: Updating mgr.smithi006.ssyogp 2024-05-20T22:23:45.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:45.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: pgmap v30: 97 pgs: 97 active+clean; 325 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 7.0 MiB/s wr, 641 op/s 2024-05-20T22:23:45.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:23:45.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:23:45.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:23:45.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:23:45.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:45 smithi078 ceph-mon[27225]: Deploying daemon mgr.smithi006.ssyogp on smithi006 2024-05-20T22:23:46.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 95269 13.62 MB/sec execute 52 sec latency 36.713 ms 2024-05-20T22:23:47.053 INFO:teuthology.orchestra.run.smithi006.stdout: 5 95717 13.62 MB/sec execute 53 sec latency 39.779 ms 2024-05-20T22:23:48.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 96154 13.71 MB/sec execute 54 sec latency 36.893 ms 2024-05-20T22:23:48.066 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:47 smithi006 ceph-mon[24790]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 643 op/s 2024-05-20T22:23:48.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:47 smithi078 ceph-mon[27225]: pgmap v31: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 643 op/s 2024-05-20T22:23:49.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 96484 13.58 MB/sec execute 55 sec latency 39.952 ms 2024-05-20T22:23:50.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 96872 13.39 MB/sec execute 56 sec latency 41.954 ms 2024-05-20T22:23:50.077 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:49 smithi006 ceph-mon[24790]: pgmap v32: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.7 MiB/s wr, 914 op/s 2024-05-20T22:23:50.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:49 smithi078 ceph-mon[27225]: pgmap v32: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.7 MiB/s wr, 914 op/s 2024-05-20T22:23:51.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 97151 13.26 MB/sec execute 57 sec latency 42.714 ms 2024-05-20T22:23:51.107 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:50 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:51.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:50 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:23:52.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 97461 13.23 MB/sec execute 58 sec latency 50.880 ms 2024-05-20T22:23:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:52 smithi006 ceph-mon[24790]: pgmap v33: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 638 op/s 2024-05-20T22:23:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:52 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:52 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:52 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:23:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:52 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:52.395 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:52 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:52.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:52 smithi078 ceph-mon[27225]: pgmap v33: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 638 op/s 2024-05-20T22:23:52.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:52 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:52.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:52 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:52.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:52 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:23:52.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:52 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:52.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:52 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:53.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 97961 13.17 MB/sec execute 59 sec latency 53.249 ms 2024-05-20T22:23:54.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 98425 13.12 MB/sec execute 60 sec latency 28.740 ms 2024-05-20T22:23:54.421 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:54 smithi078 ceph-mon[27225]: pgmap v34: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 880 op/s 2024-05-20T22:23:54.425 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:54 smithi006 ceph-mon[24790]: pgmap v34: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.0 MiB/s wr, 880 op/s 2024-05-20T22:23:55.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 98825 13.19 MB/sec execute 61 sec latency 40.531 ms 2024-05-20T22:23:56.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 99231 13.17 MB/sec execute 62 sec latency 42.532 ms 2024-05-20T22:23:56.445 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:56 smithi006 ceph-mon[24790]: pgmap v35: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.1 MiB/s wr, 514 op/s 2024-05-20T22:23:56.445 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:56 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:56.445 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:56 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:56.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:56 smithi078 ceph-mon[27225]: pgmap v35: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.1 MiB/s wr, 514 op/s 2024-05-20T22:23:56.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:56 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:56.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:56 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:23:57.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 99575 13.17 MB/sec execute 63 sec latency 46.084 ms 2024-05-20T22:23:58.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 99809 13.09 MB/sec execute 64 sec latency 47.888 ms 2024-05-20T22:23:58.468 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:23:58 smithi006 ceph-mon[24790]: pgmap v36: 97 pgs: 97 active+clean; 315 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 516 op/s 2024-05-20T22:23:58.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:23:58 smithi078 ceph-mon[27225]: pgmap v36: 97 pgs: 97 active+clean; 315 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.1 MiB/s wr, 516 op/s 2024-05-20T22:23:59.054 INFO:teuthology.orchestra.run.smithi006.stdout: 5 100073 12.97 MB/sec execute 65 sec latency 46.641 ms 2024-05-20T22:24:00.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 100476 12.82 MB/sec execute 66 sec latency 57.868 ms 2024-05-20T22:24:00.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:00 smithi006 ceph-mon[24790]: pgmap v37: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.6 MiB/s wr, 744 op/s 2024-05-20T22:24:00.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:00 smithi078 ceph-mon[27225]: pgmap v37: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 7.6 MiB/s wr, 744 op/s 2024-05-20T22:24:01.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 100748 12.72 MB/sec execute 67 sec latency 43.049 ms 2024-05-20T22:24:02.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 101151 12.75 MB/sec execute 68 sec latency 42.839 ms 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: pgmap v38: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 472 op/s 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: Standby manager daemon smithi006.ssyogp restarted 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: Standby manager daemon smithi006.ssyogp started 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/crt"}]: dispatch 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/key"}]: dispatch 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:02.501 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:02 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:02.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: pgmap v38: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 472 op/s 2024-05-20T22:24:02.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: Standby manager daemon smithi006.ssyogp restarted 2024-05-20T22:24:02.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: Standby manager daemon smithi006.ssyogp started 2024-05-20T22:24:02.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/crt"}]: dispatch 2024-05-20T22:24:02.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:24:02.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi006.ssyogp/key"}]: dispatch 2024-05-20T22:24:02.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:24:02.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:02.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:02 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:03.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 101603 12.67 MB/sec execute 69 sec latency 44.905 ms 2024-05-20T22:24:03.507 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:03 smithi006 ceph-mon[24790]: mgrmap e28: smithi078.vxzcbt(active, since 72s), standbys: smithi006.ssyogp 2024-05-20T22:24:03.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:03 smithi078 ceph-mon[27225]: mgrmap e28: smithi078.vxzcbt(active, since 72s), standbys: smithi006.ssyogp 2024-05-20T22:24:04.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 101971 12.61 MB/sec execute 70 sec latency 46.020 ms 2024-05-20T22:24:04.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:04 smithi006 ceph-mon[24790]: pgmap v39: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 679 op/s 2024-05-20T22:24:04.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:04 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:04.538 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:04 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:04.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:04 smithi078 ceph-mon[27225]: pgmap v39: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 679 op/s 2024-05-20T22:24:04.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:04 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:04.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:04 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:05.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 102301 12.64 MB/sec execute 71 sec latency 41.959 ms 2024-05-20T22:24:05.706 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:24:06.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 102755 12.68 MB/sec execute 72 sec latency 34.997 ms 2024-05-20T22:24:06.321 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:06 smithi006 ceph-mon[24790]: pgmap v40: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 437 op/s 2024-05-20T22:24:06.321 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:06 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:06 smithi078 ceph-mon[27225]: pgmap v40: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 437 op/s 2024-05-20T22:24:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:06 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:06.878 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:24:06.878 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (49s) 5s ago 9m 15.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:24:06.878 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (9m) 5s ago 9m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:24:06.878 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (8m) 11s ago 8m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:24:06.878 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (41s) 5s ago 9m 43.6M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:24:06.878 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (3m) 5s ago 3m 35.9M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (3m) 11s ago 3m 12.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (16s) 5s ago 10m 125M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (95s) 11s ago 8m 507M - 19.0.0-2695-g8a373da3 86045c3f4e75 ab4289c5f48c 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (11m) 5s ago 11m 53.9M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (8m) 11s ago 8m 39.4M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (3m) 5s ago 3m 257M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (3m) 11s ago 3m 81.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (37s) 5s ago 9m 12.2M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (25s) 11s ago 8m 10.9M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (8m) 5s ago 8m 385M 1937M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (7m) 5s ago 7m 287M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (7m) 5s ago 7m 236M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:24:06.879 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (6m) 5s ago 6m 358M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:24:06.880 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (6m) 11s ago 6m 356M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:24:06.880 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (6m) 11s ago 6m 386M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:24:06.880 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (5m) 11s ago 5m 319M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:24:06.880 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (5m) 11s ago 5m 478M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:24:06.880 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (28s) 5s ago 8m 52.5M - 2.33.4 514e6a882f6e 5b38f940880d 2024-05-20T22:24:07.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 103173 12.70 MB/sec execute 73 sec latency 40.028 ms 2024-05-20T22:24:07.329 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:07 smithi006 ceph-mon[24790]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:07.329 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:07 smithi006 ceph-mon[24790]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:24:07.541 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:24:07.542 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:24:07.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:07 smithi078 ceph-mon[27225]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:07.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:07 smithi078 ceph-mon[27225]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:08.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 103439 12.64 MB/sec execute 74 sec latency 34.016 ms 2024-05-20T22:24:08.092 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:24:08.092 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:24:08.092 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:24:08.092 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:24:08.092 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:24:08.093 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:24:08.093 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:24:08.093 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "2/23 daemons upgraded", 2024-05-20T22:24:08.093 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mgr daemons", 2024-05-20T22:24:08.093 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:24:08.093 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:24:08.343 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:08 smithi006 ceph-mon[24790]: pgmap v41: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-05-20T22:24:08.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:08 smithi006 ceph-mon[24790]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:08.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:08 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2653907497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:24:08.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:08 smithi006 ceph-mon[24790]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:08.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:08 smithi078 ceph-mon[27225]: pgmap v41: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-05-20T22:24:08.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:08 smithi078 ceph-mon[27225]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:08.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:08 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2653907497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:24:08.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:08 smithi078 ceph-mon[27225]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:08.774 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:24:09.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 103837 12.53 MB/sec execute 75 sec latency 38.014 ms 2024-05-20T22:24:09.355 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: pgmap v42: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.9 MiB/s wr, 663 op/s 2024-05-20T22:24:09.355 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/402635293' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:24:09.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:09.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:09.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:09.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:24:09.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:09.356 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:09 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:09.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: pgmap v42: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.9 MiB/s wr, 663 op/s 2024-05-20T22:24:09.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/402635293' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:24:09.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:09.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:09.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:09.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:24:09.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:09.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:09 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' 2024-05-20T22:24:10.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 104165 12.43 MB/sec execute 76 sec latency 34.480 ms 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: Failing over to other MGR 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr fail", "who": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr fail", "who": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: osdmap e71: 8 total, 8 up, 8 in 2024-05-20T22:24:10.421 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:10 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:24:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:24:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: Upgrade: Need to upgrade myself (mgr.smithi078.vxzcbt) 2024-05-20T22:24:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: Failing over to other MGR 2024-05-20T22:24:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: from='mgr.24457 172.21.15.78:0/670534022' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr fail", "who": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "mgr fail", "who": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: osdmap e71: 8 total, 8 up, 8 in 2024-05-20T22:24:10.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:10 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:24:11.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 104425 12.37 MB/sec execute 77 sec latency 50.062 ms 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd='[{"prefix": "mgr fail", "who": "smithi078.vxzcbt"}]': finished 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: mgrmap e29: smithi006.ssyogp(active, starting, since 0.978755s), standbys: smithi078.vxzcbt 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:11.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:11.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:11.459 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:24:11.459 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:11 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24457 ' entity='mgr.smithi078.vxzcbt' cmd='[{"prefix": "mgr fail", "who": "smithi078.vxzcbt"}]': finished 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: mgrmap e29: smithi006.ssyogp(active, starting, since 0.978755s), standbys: smithi078.vxzcbt 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:24:11.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:24:11.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:11 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:24:12.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 104767 12.36 MB/sec execute 78 sec latency 52.447 ms 2024-05-20T22:24:12.470 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:12 smithi006 ceph-mon[24790]: mgrmap e30: smithi006.ssyogp(active, since 1.99721s), standbys: smithi078.vxzcbt 2024-05-20T22:24:12.470 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:12 smithi006 ceph-mon[24790]: pgmap v3: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:12.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:12 smithi078 ceph-mon[27225]: mgrmap e30: smithi006.ssyogp(active, since 1.99721s), standbys: smithi078.vxzcbt 2024-05-20T22:24:12.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:12 smithi078 ceph-mon[27225]: pgmap v3: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:13.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 105257 12.29 MB/sec execute 79 sec latency 48.254 ms 2024-05-20T22:24:13.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:13 smithi006 ceph-mon[24790]: pgmap v4: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:13.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:13 smithi006 ceph-mon[24790]: mgrmap e31: smithi006.ssyogp(active, since 3s), standbys: smithi078.vxzcbt 2024-05-20T22:24:13.504 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:13 smithi078 ceph-mon[27225]: pgmap v4: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:13.504 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:13 smithi078 ceph-mon[27225]: mgrmap e31: smithi006.ssyogp(active, since 3s), standbys: smithi078.vxzcbt 2024-05-20T22:24:14.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 105581 12.25 MB/sec execute 80 sec latency 38.879 ms 2024-05-20T22:24:14.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:14 smithi006 ceph-mon[24790]: [20/May/2024:22:24:13] ENGINE Bus STARTING 2024-05-20T22:24:14.506 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:14 smithi006 ceph-mon[24790]: [20/May/2024:22:24:13] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:24:14.506 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:14 smithi006 ceph-mon[24790]: [20/May/2024:22:24:13] ENGINE Serving on http://172.21.15.6:8765 2024-05-20T22:24:14.506 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:14 smithi006 ceph-mon[24790]: [20/May/2024:22:24:13] ENGINE Bus STARTED 2024-05-20T22:24:14.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:14 smithi078 ceph-mon[27225]: [20/May/2024:22:24:13] ENGINE Bus STARTING 2024-05-20T22:24:14.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:14 smithi078 ceph-mon[27225]: [20/May/2024:22:24:13] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:24:14.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:14 smithi078 ceph-mon[27225]: [20/May/2024:22:24:13] ENGINE Serving on http://172.21.15.6:8765 2024-05-20T22:24:14.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:14 smithi078 ceph-mon[27225]: [20/May/2024:22:24:13] ENGINE Bus STARTED 2024-05-20T22:24:15.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 105956 12.31 MB/sec execute 81 sec latency 78.776 ms 2024-05-20T22:24:15.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:15 smithi006 ceph-mon[24790]: pgmap v5: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:15.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:15 smithi006 ceph-mon[24790]: mgrmap e32: smithi006.ssyogp(active, since 5s), standbys: smithi078.vxzcbt 2024-05-20T22:24:15.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:15 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:15.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:15 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:15.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:15 smithi078 ceph-mon[27225]: pgmap v5: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:15.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:15 smithi078 ceph-mon[27225]: mgrmap e32: smithi006.ssyogp(active, since 5s), standbys: smithi078.vxzcbt 2024-05-20T22:24:15.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:15 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:15.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:15 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:16.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 106532 12.40 MB/sec execute 82 sec latency 34.348 ms 2024-05-20T22:24:17.055 INFO:teuthology.orchestra.run.smithi006.stdout: 5 106929 12.41 MB/sec execute 83 sec latency 53.333 ms 2024-05-20T22:24:17.798 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:17 smithi006 ceph-mon[24790]: pgmap v6: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:17.799 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:17 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:17.799 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:17 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:17.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:17 smithi078 ceph-mon[27225]: pgmap v6: 97 pgs: 97 active+clean; 318 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-05-20T22:24:17.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:17 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:17.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:17 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:18.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 107471 12.34 MB/sec execute 84 sec latency 31.041 ms 2024-05-20T22:24:18.803 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:18 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:24:18.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:18 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:24:18.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:18 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:24:18.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:18 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:24:18.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:18 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:24:18.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:18 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:24:18.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:18 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:24:18.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:18 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:24:18.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:18 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:24:18.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:18 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/1763744151' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:24:18.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:18 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:24:18.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:18 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:24:19.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 107932 12.33 MB/sec execute 85 sec latency 34.511 ms 2024-05-20T22:24:19.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:19 smithi006 ceph-mon[24790]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-05-20T22:24:19.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:19 smithi006 ceph-mon[24790]: mgrmap e33: smithi006.ssyogp(active, since 9s), standbys: smithi078.vxzcbt 2024-05-20T22:24:19.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:19 smithi078 ceph-mon[27225]: pgmap v7: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 4.5 MiB/s wr, 424 op/s 2024-05-20T22:24:19.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:19 smithi078 ceph-mon[27225]: mgrmap e33: smithi006.ssyogp(active, since 9s), standbys: smithi078.vxzcbt 2024-05-20T22:24:20.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 108492 12.37 MB/sec execute 86 sec latency 43.751 ms 2024-05-20T22:24:21.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 109054 12.34 MB/sec execute 87 sec latency 28.459 ms 2024-05-20T22:24:21.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:21 smithi006 ceph-mon[24790]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 MiB/s wr, 329 op/s 2024-05-20T22:24:21.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:21 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:21.873 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:21 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:22.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 109356 12.33 MB/sec execute 88 sec latency 55.983 ms 2024-05-20T22:24:22.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:21 smithi078 ceph-mon[27225]: pgmap v8: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.5 MiB/s wr, 329 op/s 2024-05-20T22:24:22.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:21 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:22.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:21 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:23.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 109741 12.35 MB/sec execute 89 sec latency 42.524 ms 2024-05-20T22:24:23.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:23 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:23.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:23 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:23.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:23 smithi006 ceph-mon[24790]: pgmap v9: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 5.9 MiB/s wr, 602 op/s 2024-05-20T22:24:23.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:23 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:23.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:23 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:23.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:23 smithi078 ceph-mon[27225]: pgmap v9: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 5.9 MiB/s wr, 602 op/s 2024-05-20T22:24:24.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 110117 12.39 MB/sec execute 90 sec latency 37.361 ms 2024-05-20T22:24:25.056 INFO:teuthology.orchestra.run.smithi006.stdout: 5 110385 12.35 MB/sec execute 91 sec latency 51.573 ms 2024-05-20T22:24:25.448 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:25 smithi006 ceph-mon[24790]: pgmap v10: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.4 MiB/s wr, 549 op/s 2024-05-20T22:24:25.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:25 smithi078 ceph-mon[27225]: pgmap v10: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.4 MiB/s wr, 549 op/s 2024-05-20T22:24:26.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 110630 12.28 MB/sec execute 92 sec latency 47.030 ms 2024-05-20T22:24:26.463 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:26 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:26 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:27.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 111021 12.17 MB/sec execute 93 sec latency 45.440 ms 2024-05-20T22:24:27.475 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:27 smithi006 ceph-mon[24790]: pgmap v11: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.4 MiB/s wr, 549 op/s 2024-05-20T22:24:27.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:27 smithi078 ceph-mon[27225]: pgmap v11: 97 pgs: 97 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.4 MiB/s wr, 549 op/s 2024-05-20T22:24:28.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 111367 12.12 MB/sec execute 94 sec latency 36.316 ms 2024-05-20T22:24:29.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 111755 12.16 MB/sec execute 95 sec latency 47.533 ms 2024-05-20T22:24:29.176 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:29.176 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:24:29.177 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:29 smithi006 ceph-mon[24790]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:24:29.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:29.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:29.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:29.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:24:29.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:24:29.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:24:29.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 7.7 MiB/s wr, 740 op/s 2024-05-20T22:24:29.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:24:29.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:24:29.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:29 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:24:30.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 112219 12.10 MB/sec execute 96 sec latency 48.213 ms 2024-05-20T22:24:30.433 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.434 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:30 smithi006 ceph-mon[24790]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-05-20T22:24:30.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:30.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:30 smithi078 ceph-mon[27225]: Reconfiguring prometheus.smithi006 (dependencies changed)... 2024-05-20T22:24:31.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 112609 12.07 MB/sec execute 97 sec latency 37.292 ms 2024-05-20T22:24:31.441 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:31 smithi006 ceph-mon[24790]: Reconfiguring daemon prometheus.smithi006 on smithi006 2024-05-20T22:24:31.441 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:31 smithi006 ceph-mon[24790]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-05-20T22:24:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:31 smithi078 ceph-mon[27225]: Reconfiguring daemon prometheus.smithi006 on smithi006 2024-05-20T22:24:31.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:31 smithi078 ceph-mon[27225]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-05-20T22:24:32.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 113030 12.11 MB/sec execute 98 sec latency 26.550 ms 2024-05-20T22:24:33.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 113483 12.15 MB/sec execute 99 sec latency 37.707 ms 2024-05-20T22:24:33.464 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:33 smithi006 ceph-mon[24790]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 743 op/s 2024-05-20T22:24:33.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:33 smithi078 ceph-mon[27225]: pgmap v14: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 743 op/s 2024-05-20T22:24:34.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 113976 12.20 MB/sec execute 100 sec latency 31.297 ms 2024-05-20T22:24:35.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 114465 12.15 MB/sec execute 101 sec latency 24.164 ms 2024-05-20T22:24:35.491 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:35 smithi006 ceph-mon[24790]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-05-20T22:24:35.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:35 smithi078 ceph-mon[27225]: pgmap v15: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-05-20T22:24:36.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 114932 12.10 MB/sec execute 102 sec latency 40.281 ms 2024-05-20T22:24:37.005 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:36 smithi006 ceph-mon[24790]: pgmap v16: 97 pgs: 97 active+clean; 322 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.0 MiB/s wr, 459 op/s 2024-05-20T22:24:37.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 115647 12.19 MB/sec execute 103 sec latency 45.632 ms 2024-05-20T22:24:37.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:36 smithi078 ceph-mon[27225]: pgmap v16: 97 pgs: 97 active+clean; 322 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.0 MiB/s wr, 459 op/s 2024-05-20T22:24:38.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 116213 12.18 MB/sec execute 104 sec latency 33.280 ms 2024-05-20T22:24:39.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 116677 12.24 MB/sec execute 105 sec latency 40.631 ms 2024-05-20T22:24:39.428 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:24:39.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:39 smithi006 ceph-mon[24790]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 732 op/s 2024-05-20T22:24:39.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:39 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:39.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:39 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:39.529 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:39 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:24:39.530 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:39 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:39.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:39 smithi078 ceph-mon[27225]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 732 op/s 2024-05-20T22:24:39.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:39 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:39.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:39 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:39.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:39 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:24:39.817 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:39 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:40.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 117188 12.30 MB/sec execute 106 sec latency 58.125 ms 2024-05-20T22:24:40.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:40 smithi006 ceph-mon[24790]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:24:40.540 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:40 smithi006 ceph-mon[24790]: from='client.14794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (83s) 19s ago 10m 15.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (10m) 19s ago 10m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (9m) 25s ago 9m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (74s) 19s ago 9m 46.2M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (4m) 19s ago 4m 36.2M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (4m) 25s ago 4m 12.3M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (50s) 19s ago 11m 480M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283 running (2m) 25s ago 9m 84.6M - 19.0.0-2695-g8a373da3 86045c3f4e75 ab4289c5f48c 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (11m) 19s ago 11m 56.1M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:24:40.555 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (9m) 25s ago 9m 42.9M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (3m) 19s ago 3m 259M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (3m) 25s ago 3m 83.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (71s) 19s ago 9m 9747k - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (59s) 25s ago 9m 12.5M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (8m) 19s ago 8m 414M 1937M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (8m) 19s ago 8m 316M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (7m) 19s ago 7m 239M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (7m) 19s ago 7m 386M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (6m) 25s ago 6m 370M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (6m) 25s ago 6m 396M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (6m) 25s ago 6m 337M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (5m) 25s ago 5m 512M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:24:40.556 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 starting - - - - 2024-05-20T22:24:40.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:40 smithi078 ceph-mon[27225]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:24:40.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:40 smithi078 ceph-mon[27225]: from='client.14794 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:41.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 117508 12.30 MB/sec execute 107 sec latency 37.774 ms 2024-05-20T22:24:41.142 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:24:41.143 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:24:41.144 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:24:41.144 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-05-20T22:24:41.144 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:24:41.144 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:24:41.144 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:24:41.548 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:41.548 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-05-20T22:24:41.549 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:41.549 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:41.549 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/2924856131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:24:41.549 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:24:41.549 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:24:41.549 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:41 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:41.597 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:24:41.597 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:24:41.597 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:24:41.597 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:24:41.597 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:24:41.598 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:24:41.598 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:24:41.598 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "2/23 daemons upgraded", 2024-05-20T22:24:41.598 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mgr daemons", 2024-05-20T22:24:41.598 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:24:41.598 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:24:41.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:41.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-05-20T22:24:41.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:41.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:41.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2924856131' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:24:41.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:24:41.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:24:41.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:41 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:42.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 117758 12.23 MB/sec execute 108 sec latency 39.765 ms 2024-05-20T22:24:42.165 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:24:42.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:42 smithi006 ceph-mon[24790]: Upgrade: Updating mgr.smithi078.vxzcbt 2024-05-20T22:24:42.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:42 smithi006 ceph-mon[24790]: Deploying daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:24:42.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:42 smithi006 ceph-mon[24790]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:42.578 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:42 smithi006 ceph-mon[24790]: from='client.? 172.21.15.6:0/890207223' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:24:42.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:42 smithi078 ceph-mon[27225]: Upgrade: Updating mgr.smithi078.vxzcbt 2024-05-20T22:24:42.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:42 smithi078 ceph-mon[27225]: Deploying daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:24:42.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:42 smithi078 ceph-mon[27225]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:24:42.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:42 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/890207223' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:24:43.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 118280 12.16 MB/sec execute 109 sec latency 43.340 ms 2024-05-20T22:24:43.584 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:43 smithi006 ceph-mon[24790]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 833 op/s 2024-05-20T22:24:43.628 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:43 smithi078 ceph-mon[27225]: pgmap v19: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.5 MiB/s wr, 833 op/s 2024-05-20T22:24:44.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 118728 12.19 MB/sec execute 110 sec latency 59.941 ms 2024-05-20T22:24:45.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 119415 12.19 MB/sec execute 111 sec latency 32.524 ms 2024-05-20T22:24:45.601 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:45 smithi006 ceph-mon[24790]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 585 op/s 2024-05-20T22:24:45.719 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:45 smithi078 ceph-mon[27225]: pgmap v20: 97 pgs: 97 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.4 MiB/s wr, 585 op/s 2024-05-20T22:24:46.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 119859 12.19 MB/sec execute 112 sec latency 41.419 ms 2024-05-20T22:24:46.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:46 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:46.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:46 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:46.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:46 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:46.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:46 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:46.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:46 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:46.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:46 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:47.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 120389 12.27 MB/sec execute 113 sec latency 38.093 ms 2024-05-20T22:24:47.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:47 smithi006 ceph-mon[24790]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 608 op/s 2024-05-20T22:24:47.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:47 smithi078 ceph-mon[27225]: pgmap v21: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 608 op/s 2024-05-20T22:24:48.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 120761 12.28 MB/sec execute 114 sec latency 39.213 ms 2024-05-20T22:24:49.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 121042 12.27 MB/sec execute 115 sec latency 54.023 ms 2024-05-20T22:24:49.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:49 smithi078 ceph-mon[27225]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.5 MiB/s wr, 856 op/s 2024-05-20T22:24:49.935 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:49 smithi006 ceph-mon[24790]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.5 MiB/s wr, 856 op/s 2024-05-20T22:24:50.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 121270 12.21 MB/sec execute 116 sec latency 61.953 ms 2024-05-20T22:24:51.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 121678 12.13 MB/sec execute 117 sec latency 69.078 ms 2024-05-20T22:24:51.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:51 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:51.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:51 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:51.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:51 smithi006 ceph-mon[24790]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 583 op/s 2024-05-20T22:24:51.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:51 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:51.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:51 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:51.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:51 smithi078 ceph-mon[27225]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 583 op/s 2024-05-20T22:24:52.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 122004 12.09 MB/sec execute 118 sec latency 27.170 ms 2024-05-20T22:24:52.709 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:52 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:52.709 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:52 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:52.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:52 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:52.733 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:52 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:53.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 122454 12.12 MB/sec execute 119 sec latency 56.909 ms 2024-05-20T22:24:53.732 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:53 smithi006 ceph-mon[24790]: pgmap v24: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 798 op/s 2024-05-20T22:24:53.873 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:53 smithi078 ceph-mon[27225]: pgmap v24: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 798 op/s 2024-05-20T22:24:54.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 122971 12.08 MB/sec execute 120 sec latency 62.964 ms 2024-05-20T22:24:55.057 INFO:teuthology.orchestra.run.smithi006.stdout: 5 123310 12.06 MB/sec execute 121 sec latency 37.843 ms 2024-05-20T22:24:55.504 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: pgmap v25: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 506 op/s 2024-05-20T22:24:55.504 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:24:55.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:55 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:55.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: pgmap v25: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 506 op/s 2024-05-20T22:24:55.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2511160158' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:24:55.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:55 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:24:56.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 123721 12.10 MB/sec execute 122 sec latency 32.096 ms 2024-05-20T22:24:56.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: mgrmap e34: smithi006.ssyogp(active, since 46s), standbys: smithi078.vxzcbt 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.ssyogp"}]: dispatch 2024-05-20T22:24:56.762 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.ssyogp"}]': finished 2024-05-20T22:24:56.763 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:56.763 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.vxzcbt"}]': finished 2024-05-20T22:24:56.763 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:56 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:24:57.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 124292 12.15 MB/sec execute 123 sec latency 51.405 ms 2024-05-20T22:24:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: mgrmap e34: smithi006.ssyogp(active, since 46s), standbys: smithi078.vxzcbt 2024-05-20T22:24:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:24:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:24:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:57.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.ssyogp"}]: dispatch 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi006.ssyogp"}]': finished 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.vxzcbt"}]: dispatch 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi078.vxzcbt"}]': finished 2024-05-20T22:24:57.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:56 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:24:57.775 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:57 smithi006 ceph-mon[24790]: Upgrade: Setting container_image for all mgr 2024-05-20T22:24:57.775 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:57 smithi006 ceph-mon[24790]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 681 op/s 2024-05-20T22:24:57.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:57 smithi078 ceph-mon[27225]: Upgrade: Setting container_image for all mgr 2024-05-20T22:24:57.974 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:57 smithi078 ceph-mon[27225]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 681 op/s 2024-05-20T22:24:58.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 124678 12.17 MB/sec execute 124 sec latency 28.256 ms 2024-05-20T22:24:59.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 125003 12.10 MB/sec execute 125 sec latency 74.618 ms 2024-05-20T22:24:59.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:59 smithi006 ceph-mon[24790]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.2 MiB/s wr, 741 op/s 2024-05-20T22:24:59.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:59 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:24:59.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:59 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:24:59.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:59 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:24:59.805 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:24:59 smithi006 ceph-mon[24790]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:00.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 125348 12.03 MB/sec execute 126 sec latency 49.860 ms 2024-05-20T22:25:00.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:59 smithi078 ceph-mon[27225]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.2 MiB/s wr, 741 op/s 2024-05-20T22:25:00.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:59 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:00.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:59 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:25:00.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:59 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:25:00.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:24:59 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:00.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:00 smithi006 ceph-mon[24790]: Upgrade: Updating mon.smithi006 2024-05-20T22:25:00.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:00 smithi006 ceph-mon[24790]: Deploying daemon mon.smithi006 on smithi006 2024-05-20T22:25:01.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 125551 11.98 MB/sec execute 127 sec latency 64.983 ms 2024-05-20T22:25:01.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:00 smithi078 ceph-mon[27225]: Upgrade: Updating mon.smithi006 2024-05-20T22:25:01.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:00 smithi078 ceph-mon[27225]: Deploying daemon mon.smithi006 on smithi006 2024-05-20T22:25:01.833 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:01 smithi006 ceph-mon[24790]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 473 op/s 2024-05-20T22:25:02.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 125938 12.00 MB/sec execute 128 sec latency 70.882 ms 2024-05-20T22:25:02.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:01 smithi078 ceph-mon[27225]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.0 MiB/s wr, 473 op/s 2024-05-20T22:25:02.603 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:02 smithi006 systemd[1]: Stopping Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:25:02.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[24786]: 2024-05-20T22:25:02.782+0000 7f42ceb81700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi006 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:25:02.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[24786]: 2024-05-20T22:25:02.782+0000 7f42ceb81700 -1 mon.smithi006@0(leader) e2 *** Got Signal Terminated *** 2024-05-20T22:25:02.854 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:02 smithi006 podman[65587]: 2024-05-20 22:25:02.85641531 +0000 UTC m=+0.258384634 container died 8dde577b64917d8e8766480b90a3fb1b9d06f0da1d83a9168fb1c1fe399551bb (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.expose-services=, vcs-type=git, distribution-scope=public, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-20T22:25:03.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 126325 11.95 MB/sec execute 129 sec latency 57.410 ms 2024-05-20T22:25:03.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:03 smithi006 podman[65587]: 2024-05-20 22:25:03.702037205 +0000 UTC m=+1.104006512 container cleanup 8dde577b64917d8e8766480b90a3fb1b9d06f0da1d83a9168fb1c1fe399551bb (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, distribution-scope=public, io.buildah.version=1.19.8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.openshift.expose-services=, release=754, GIT_CLEAN=True) 2024-05-20T22:25:03.859 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:03 smithi006 bash[65587]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006 2024-05-20T22:25:04.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 126713 11.93 MB/sec execute 130 sec latency 47.046 ms 2024-05-20T22:25:04.111 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:04 smithi006 podman[65610]: 2024-05-20 22:25:04.013236543 +0000 UTC m=+1.158238376 container remove 8dde577b64917d8e8766480b90a3fb1b9d06f0da1d83a9168fb1c1fe399551bb (image=quay.io/ceph/ceph:v17.2.0, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., name=centos-stream, RELEASE=HEAD, version=8, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, release=754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:25:04.636 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:04 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service: Deactivated successfully. 2024-05-20T22:25:04.636 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:04 smithi006 systemd[1]: Stopped Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:25:04.637 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:04 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service: Consumed 17.364s CPU time. 2024-05-20T22:25:04.890 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:04 smithi006 systemd[1]: Starting Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:25:05.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 126935 11.88 MB/sec execute 131 sec latency 45.143 ms 2024-05-20T22:25:05.147 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 podman[65734]: 2024-05-20 22:25:04.913725468 +0000 UTC m=+0.026711418 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:25:05.147 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 podman[65734]: 2024-05-20 22:25:05.030040273 +0000 UTC m=+0.143026208 container create 7db33fd7bf1f0a4b5a63a458e17a2825bd89f094559ec049fc9a0d7093911db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-20T22:25:05.655 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 podman[65734]: 2024-05-20 22:25:05.508544397 +0000 UTC m=+0.621530341 container init 7db33fd7bf1f0a4b5a63a458e17a2825bd89f094559ec049fc9a0d7093911db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 podman[65734]: 2024-05-20 22:25:05.514454955 +0000 UTC m=+0.627440895 container start 7db33fd7bf1f0a4b5a63a458e17a2825bd89f094559ec049fc9a0d7093911db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD) 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev), process ceph-mon, pid 2 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: pidfile_write: ignore empty --pid-file 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: load: jerasure load: lrc 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: RocksDB version: 7.9.2 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Git sha 0 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Compile date 2024-05-17 12:39:40 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: DB SUMMARY 2024-05-20T22:25:05.656 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: DB Session ID: 8T9SBO2RKLA0EVQT1QWH 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi006/store.db dir, Total Num: 1, files: 000030.sst 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi006/store.db: 000028.log size: 1038928 ; 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.env: 0x55bb9e0dbc60 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.fs: PosixFileSystem 2024-05-20T22:25:05.657 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.info_log: 0x55bba02e9980 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.statistics: (nil) 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.use_fsync: 0 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:25:05.658 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.db_log_dir: 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.wal_dir: 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:25:05.659 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.write_buffer_manager: 0x55bba02bc1e0 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.unordered_write: 0 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:25:05.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:25:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.row_cache: None 2024-05-20T22:25:05.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.wal_filter: None 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.wal_compression: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:25:05.662 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:25:05.663 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_open_files: -1 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Compression algorithms supported: 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:25:05.664 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kZSTD supported: 0 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000015 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.merge_operator: 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_filter: None 2024-05-20T22:25:05.665 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bba02e9a80) 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_type: 0 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_index_type: 0 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_shortening: 1 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: checksum: 4 2024-05-20T22:25:05.666 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: no_block_cache: 0 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache: 0x55bba02ed090 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_options: 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: capacity : 536870912 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: num_shard_bits : 4 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: strict_capacity_limit : 0 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_cache_compressed: (nil) 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: persistent_cache: (nil) 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size: 4096 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_size_deviation: 10 2024-05-20T22:25:05.667 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_restart_interval: 16 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: index_block_restart_interval: 1 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: metadata_block_size: 4096 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: partition_filters: 0 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: use_delta_encoding: 1 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: filter_policy: bloomfilter 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: whole_key_filtering: 1 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: verify_compression: 0 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: format_version: 5 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_index_compression: 1 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: block_align: 0 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_auto_readahead_size: 262144 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: prepopulate_block_cache: 0 2024-05-20T22:25:05.668 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: initial_auto_readahead_size: 8192 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: num_file_reads_for_auto_readahead: 2 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression: NoCompression 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.num_levels: 7 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:25:05.669 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:25:05.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:25:05.671 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.arena_block_size: 1048576 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:25:05.672 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:25:05.673 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.ttl: 2592000 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-20T22:25:05.674 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.blob_file_starting_level: 0 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi006/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11876, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a2236df6-5be1-446b-928d-5a7ef87f2516 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243905549343, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243905554700, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 931885, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11877, "largest_seqno": 12530, "table_properties": {"data_size": 928155, "index_size": 1948, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 6705, "raw_average_key_size": 23, "raw_value_size": 921784, "raw_average_value_size": 3257, "num_data_blocks": 88, "num_entries": 283, "num_filter_entries": 283, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716243905, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a2236df6-5be1-446b-928d-5a7ef87f2516", "db_session_id": "8T9SBO2RKLA0EVQT1QWH", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243905554810, "job": 1, "event": "recovery_finished"} 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-05-20T22:25:05.675 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi006/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bba03de000 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: DB pointer 0x55bba03c8000 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** DB Stats ** 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:25:05.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L0 1/0 910.04 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 324.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: L6 1/0 7.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Sum 2/0 8.54 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 324.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 324.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** Compaction Stats [default] ** 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 324.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Cumulative compaction: 0.00 GB write, 92.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Interval compaction: 0.00 GB write, 92.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Block cache BinnedLRUCache@0x55bba02ed090#2 capacity: 512.00 MB usage: 2.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-05-20T22:25:05.678 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,2.08 KB,0.000396371%) Misc(1,0.00 KB,0%) 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: starting mon.smithi006 rank 0 at public addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] at bind addrs [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi006 fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???) e2 preinit fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???).mds e7 new map 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???).mds e7 print_map 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: e7 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: legacy client fscid: 1 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Filesystem 'foofs' (1) 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: fs_name foofs 2024-05-20T22:25:05.679 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: epoch 5 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: created 2024-05-20T22:20:16.358680+0000 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: modified 2024-05-20T22:20:21.384193+0000 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: tableserver 0 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: root 0 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: session_timeout 60 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: session_autoclose 300 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_file_size 1099511627776 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_xattr_size 65536 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: required_client_features {} 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: last_failure 0 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: last_failure_osd_epoch 0 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:25:05.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: max_mds 1 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: in 0 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: up {0=14476} 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: failed 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: damaged 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: stopped 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: data_pools [3] 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: metadata_pool 2 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: inline_data disabled 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: balancer 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: bal_rank_mask -1 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: standby_count_wanted 1 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: qdb_cluster leader: 0 members: 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: [mds.foofs.smithi006.pburlb{0:14476} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.6:6834/3827922570,v1:172.21.15.6:6835/3827922570] compat {c=[1],r=[1],i=[7ff]}] 2024-05-20T22:25:05.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: Standby daemons: 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout: [mds.foofs.smithi078.yocghn{-1:24311} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] compat {c=[1],r=[1],i=[7ff]}] 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 ceph-mon[65753]: mon.smithi006@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 bash[65734]: 7db33fd7bf1f0a4b5a63a458e17a2825bd89f094559ec049fc9a0d7093911db8 2024-05-20T22:25:05.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:05 smithi006 systemd[1]: Started Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:25:06.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 127371 11.92 MB/sec execute 132 sec latency 32.295 ms 2024-05-20T22:25:06.416 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:25:06.416 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:25:06.416 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 488 op/s 2024-05-20T22:25:06.416 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: mon.smithi006 calling monitor election 2024-05-20T22:25:06.416 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: mon.smithi006 is new leader, mons smithi006,smithi078 in quorum (ranks 0,1) 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: monmap epoch 2 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: last_changed 2024-05-20T22:15:17.269330+0000 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: created 2024-05-20T22:12:48.663636+0000 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: min_mon_release 17 (quincy) 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: election_strategy: 1 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: 1: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: osdmap e71: 8 total, 8 up, 8 in 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: mgrmap e34: smithi006.ssyogp(active, since 56s), standbys: smithi078.vxzcbt 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: overall HEALTH_OK 2024-05-20T22:25:06.417 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: mgrmap e35: smithi006.ssyogp(active, since 56s), standbys: smithi078.vxzcbt 2024-05-20T22:25:06.418 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:06 smithi006 ceph-mon[65753]: from='mgr.24525 ' entity='' 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: from='mgr.24525 172.21.15.6:0/696617803' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 488 op/s 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: mon.smithi006 calling monitor election 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: mon.smithi006 is new leader, mons smithi006,smithi078 in quorum (ranks 0,1) 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: monmap epoch 2 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:06.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: last_changed 2024-05-20T22:15:17.269330+0000 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: created 2024-05-20T22:12:48.663636+0000 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: min_mon_release 17 (quincy) 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: election_strategy: 1 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: 1: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: osdmap e71: 8 total, 8 up, 8 in 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: mgrmap e34: smithi006.ssyogp(active, since 56s), standbys: smithi078.vxzcbt 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: overall HEALTH_OK 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: mgrmap e35: smithi006.ssyogp(active, since 56s), standbys: smithi078.vxzcbt 2024-05-20T22:25:06.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:06 smithi078 ceph-mon[27225]: from='mgr.24525 ' entity='' 2024-05-20T22:25:07.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 127789 11.95 MB/sec execute 133 sec latency 63.238 ms 2024-05-20T22:25:08.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 128045 11.94 MB/sec execute 134 sec latency 45.496 ms 2024-05-20T22:25:08.207 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:08 smithi006 ceph-mon[65753]: mgrmap e36: smithi006.ssyogp(active, since 57s), standbys: smithi078.vxzcbt 2024-05-20T22:25:08.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:08 smithi078 ceph-mon[27225]: mgrmap e36: smithi006.ssyogp(active, since 57s), standbys: smithi078.vxzcbt 2024-05-20T22:25:09.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 128359 11.91 MB/sec execute 135 sec latency 48.607 ms 2024-05-20T22:25:10.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 128800 11.84 MB/sec execute 136 sec latency 34.015 ms 2024-05-20T22:25:11.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129048 11.79 MB/sec execute 137 sec latency 75.860 ms 2024-05-20T22:25:12.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129277 11.77 MB/sec execute 138 sec latency 69.082 ms 2024-05-20T22:25:13.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129624 11.74 MB/sec execute 139 sec latency 101.827 ms 2024-05-20T22:25:14.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 129955 11.71 MB/sec execute 140 sec latency 69.290 ms 2024-05-20T22:25:15.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 130258 11.66 MB/sec execute 141 sec latency 40.632 ms 2024-05-20T22:25:15.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:15 smithi006 ceph-mon[65753]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:25:15.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:15 smithi006 ceph-mon[65753]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:25:15.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:15 smithi006 ceph-mon[65753]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:25:15.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:15 smithi006 ceph-mon[65753]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:25:15.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:15 smithi006 ceph-mon[65753]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:25:15.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:15 smithi006 ceph-mon[65753]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:25:15.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:15 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt restarted 2024-05-20T22:25:15.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:15 smithi078 ceph-mon[27225]: Standby manager daemon smithi078.vxzcbt started 2024-05-20T22:25:15.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:15 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/crt"}]: dispatch 2024-05-20T22:25:15.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:15 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-05-20T22:25:15.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:15 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi078.vxzcbt/key"}]: dispatch 2024-05-20T22:25:15.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:15 smithi078 ceph-mon[27225]: from='mgr.? 172.21.15.78:0/2290614959' entity='mgr.smithi078.vxzcbt' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-05-20T22:25:16.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 130606 11.68 MB/sec execute 142 sec latency 48.405 ms 2024-05-20T22:25:16.370 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: mgrmap e37: smithi006.ssyogp(active, since 65s), standbys: smithi078.vxzcbt 2024-05-20T22:25:16.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: Active manager daemon smithi006.ssyogp restarted 2024-05-20T22:25:16.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: Activating manager daemon smithi006.ssyogp 2024-05-20T22:25:16.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:25:16.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: mgrmap e38: smithi006.ssyogp(active, starting, since 0.010148s), standbys: smithi078.vxzcbt 2024-05-20T22:25:16.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:25:16.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:25:16.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:25:16.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:25:16.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:25:16.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:25:16.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:25:16.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:25:16.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:25:16.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:25:16.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:25:16.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:25:16.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:25:16.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:25:16.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:25:16.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:25:16.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:25:16.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:25:16.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:25:16.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:25:16.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:25:16.375 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:25:16.456 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:25:16.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: mgrmap e37: smithi006.ssyogp(active, since 65s), standbys: smithi078.vxzcbt 2024-05-20T22:25:16.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: Active manager daemon smithi006.ssyogp restarted 2024-05-20T22:25:16.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: Activating manager daemon smithi006.ssyogp 2024-05-20T22:25:16.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:25:16.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: mgrmap e38: smithi006.ssyogp(active, starting, since 0.010148s), standbys: smithi078.vxzcbt 2024-05-20T22:25:16.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:25:16.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi006.ssyogp", "id": "smithi006.ssyogp"}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr metadata", "who": "smithi078.vxzcbt", "id": "smithi078.vxzcbt"}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-05-20T22:25:16.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-05-20T22:25:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: Manager daemon smithi006.ssyogp is now available 2024-05-20T22:25:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:25:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:25:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/mirror_snapshot_schedule"}]: dispatch 2024-05-20T22:25:16.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:16 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi006.ssyogp/trash_purge_schedule"}]: dispatch 2024-05-20T22:25:17.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 130918 11.68 MB/sec execute 143 sec latency 41.406 ms 2024-05-20T22:25:17.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:17 smithi006 ceph-mon[65753]: [20/May/2024:22:25:16] ENGINE Bus STARTING 2024-05-20T22:25:17.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:17 smithi006 ceph-mon[65753]: [20/May/2024:22:25:16] ENGINE Serving on http://172.21.15.6:8765 2024-05-20T22:25:17.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:17 smithi006 ceph-mon[65753]: mgrmap e39: smithi006.ssyogp(active, since 1.02309s), standbys: smithi078.vxzcbt 2024-05-20T22:25:17.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:17.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:17.469 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:25:17.469 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (119s) 22s ago 10m 15.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:25:17.469 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (10m) 22s ago 10m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:25:17.469 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (10m) 27s ago 10m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:25:17.469 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (111s) 22s ago 10m 45.4M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:25:17.469 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (4m) 22s ago 4m 36.2M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:25:17.469 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (4m) 27s ago 4m 12.4M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (87s) 22s ago 12m 497M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (32s) 27s ago 10m 64.0M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (12m) 22s ago 12m 57.4M 2048M 17.2.0 e1d6a67b021e 8dde577b6491 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (10m) 27s ago 10m 44.2M 2048M 17.2.0 e1d6a67b021e e4bbcbbe0026 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (4m) 22s ago 4m 268M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (4m) 27s ago 4m 83.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (107s) 22s ago 10m 9193k - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (96s) 27s ago 9m 8908k - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (9m) 22s ago 9m 452M 1937M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (8m) 22s ago 8m 388M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (8m) 22s ago 8m 268M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (7m) 22s ago 7m 414M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:25:17.470 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (7m) 27s ago 7m 414M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:25:17.471 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (7m) 27s ago 7m 445M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:25:17.471 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (6m) 27s ago 6m 400M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:25:17.471 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (6m) 27s ago 6m 550M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:25:17.471 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (39s) 22s ago 9m 53.6M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:25:17.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:17 smithi078 ceph-mon[27225]: [20/May/2024:22:25:16] ENGINE Bus STARTING 2024-05-20T22:25:17.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:17 smithi078 ceph-mon[27225]: [20/May/2024:22:25:16] ENGINE Serving on http://172.21.15.6:8765 2024-05-20T22:25:17.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:17 smithi078 ceph-mon[27225]: mgrmap e39: smithi006.ssyogp(active, since 1.02309s), standbys: smithi078.vxzcbt 2024-05-20T22:25:17.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:17 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:17.567 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:17 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:18.058 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131333 11.70 MB/sec execute 144 sec latency 44.282 ms 2024-05-20T22:25:18.085 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:25:18.085 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:25:18.085 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T22:25:18.085 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:25:18.085 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:18.086 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:25:18.087 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-05-20T22:25:18.087 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:25:18.087 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:25:18.087 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:25:18.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:18 smithi006 ceph-mon[65753]: [20/May/2024:22:25:16] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:25:18.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:18 smithi006 ceph-mon[65753]: [20/May/2024:22:25:16] ENGINE Bus STARTED 2024-05-20T22:25:18.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:18 smithi006 ceph-mon[65753]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:18.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:18 smithi006 ceph-mon[65753]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:18.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:18 smithi006 ceph-mon[65753]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:18.406 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:18 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2386195806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:25:18.492 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:18 smithi078 ceph-mon[27225]: [20/May/2024:22:25:16] ENGINE Serving on https://172.21.15.6:7150 2024-05-20T22:25:18.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:18 smithi078 ceph-mon[27225]: [20/May/2024:22:25:16] ENGINE Bus STARTED 2024-05-20T22:25:18.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:18 smithi078 ceph-mon[27225]: from='client.34108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:18.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:18 smithi078 ceph-mon[27225]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:18.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:18 smithi078 ceph-mon[27225]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:18.493 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:18 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/2386195806' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:25:18.578 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "2/23 daemons upgraded", 2024-05-20T22:25:18.579 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-05-20T22:25:18.579 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:25:18.579 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:25:19.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131595 11.69 MB/sec execute 145 sec latency 39.170 ms 2024-05-20T22:25:19.168 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:25:19.180 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:19 smithi006 ceph-mon[65753]: pgmap v4: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:19.181 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:19 smithi006 ceph-mon[65753]: from='client.24619 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:19.181 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:19 smithi006 ceph-mon[65753]: mgrmap e40: smithi006.ssyogp(active, since 2s), standbys: smithi078.vxzcbt 2024-05-20T22:25:19.430 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:19 smithi078 ceph-mon[27225]: pgmap v4: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:19.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:19 smithi078 ceph-mon[27225]: from='client.24619 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:19.431 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:19 smithi078 ceph-mon[27225]: mgrmap e40: smithi006.ssyogp(active, since 2s), standbys: smithi078.vxzcbt 2024-05-20T22:25:20.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 131935 11.65 MB/sec execute 146 sec latency 34.208 ms 2024-05-20T22:25:20.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:20 smithi006 ceph-mon[65753]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:20.195 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:20 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/55966465' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:25:20.511 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:20 smithi078 ceph-mon[27225]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:20.511 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:20 smithi078 ceph-mon[27225]: from='client.? 172.21.15.6:0/55966465' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:25:21.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132294 11.60 MB/sec execute 147 sec latency 43.065 ms 2024-05-20T22:25:21.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:21 smithi006 ceph-mon[65753]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:21.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:21 smithi006 ceph-mon[65753]: mgrmap e41: smithi006.ssyogp(active, since 4s), standbys: smithi078.vxzcbt 2024-05-20T22:25:21.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:21.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:21.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:21 smithi078 ceph-mon[27225]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:21.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:21 smithi078 ceph-mon[27225]: mgrmap e41: smithi006.ssyogp(active, since 4s), standbys: smithi078.vxzcbt 2024-05-20T22:25:21.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:21 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:21.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:21 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:22.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132515 11.55 MB/sec execute 148 sec latency 58.183 ms 2024-05-20T22:25:23.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132736 11.53 MB/sec execute 149 sec latency 51.746 ms 2024-05-20T22:25:23.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:23 smithi006 ceph-mon[65753]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:23.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:23.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:23.465 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:23 smithi078 ceph-mon[27225]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-05-20T22:25:23.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:23 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:23.466 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:23 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:24.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 132961 11.49 MB/sec execute 150 sec latency 126.975 ms 2024-05-20T22:25:25.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 133350 11.47 MB/sec execute 151 sec latency 81.535 ms 2024-05-20T22:25:25.204 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:25 smithi006 ceph-mon[65753]: pgmap v7: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-05-20T22:25:25.476 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:25 smithi078 ceph-mon[27225]: pgmap v7: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-05-20T22:25:26.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 133616 11.42 MB/sec execute 152 sec latency 97.946 ms 2024-05-20T22:25:26.488 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:26.488 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:26.488 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:26 smithi006 ceph-mon[65753]: pgmap v8: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 283 op/s 2024-05-20T22:25:26.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:26 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:26.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:26 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:26.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:26 smithi078 ceph-mon[27225]: pgmap v8: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 283 op/s 2024-05-20T22:25:27.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 134128 11.46 MB/sec execute 153 sec latency 61.982 ms 2024-05-20T22:25:27.960 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:27.960 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:27.960 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:27.960 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:27.960 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:27.960 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:25:28.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 134571 11.49 MB/sec execute 154 sec latency 26.424 ms 2024-05-20T22:25:28.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:27 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:28.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:27 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:28.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:27 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:28.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:27 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:28.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:27 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:28.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:27 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:25:29.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 134969 11.52 MB/sec execute 155 sec latency 60.873 ms 2024-05-20T22:25:29.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:25:29.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:25:29.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 420 op/s 2024-05-20T22:25:29.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.495 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:25:29.496 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:25:29.496 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.conf 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.conf 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: pgmap v9: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 420 op/s 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.conf 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: Updating smithi078:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: Updating smithi006:/etc/ceph/ceph.client.admin.keyring 2024-05-20T22:25:29.527 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: Updating smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:25:29.528 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:29 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-05-20T22:25:30.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 135261 11.50 MB/sec execute 156 sec latency 40.577 ms 2024-05-20T22:25:30.505 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:30 smithi006 ceph-mon[65753]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:25:30.530 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:30 smithi078 ceph-mon[27225]: Updating smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/config/ceph.client.admin.keyring 2024-05-20T22:25:31.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 135531 11.44 MB/sec execute 157 sec latency 72.055 ms 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.6 MiB/s wr, 383 op/s 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: Upgrade: Updating mon.smithi078 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:31.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:31 smithi006 ceph-mon[65753]: Deploying daemon mon.smithi078 on smithi078 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: pgmap v10: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.6 MiB/s wr, 383 op/s 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: Upgrade: Updating mon.smithi078 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:31.586 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:31 smithi078 ceph-mon[27225]: Deploying daemon mon.smithi078 on smithi078 2024-05-20T22:25:32.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 135886 11.41 MB/sec execute 158 sec latency 51.787 ms 2024-05-20T22:25:32.274 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:32 smithi006 ceph-mon[65753]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.6 MiB/s wr, 383 op/s 2024-05-20T22:25:32.537 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:32 smithi078 ceph-mon[27225]: pgmap v11: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.6 MiB/s wr, 383 op/s 2024-05-20T22:25:33.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 136262 11.41 MB/sec execute 159 sec latency 46.084 ms 2024-05-20T22:25:33.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:32 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:25:33.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:33 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078[27219]: 2024-05-20T22:25:33.153+0000 7f3961568700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:25:33.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:33 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078[27219]: 2024-05-20T22:25:33.153+0000 7f3961568700 -1 mon.smithi078@1(peon) e2 *** Got Signal Terminated *** 2024-05-20T22:25:33.485 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:33 smithi078 podman[52762]: 2024-05-20 22:25:33.1849707 +0000 UTC m=+0.171835161 container died e4bbcbbe002698380a4c0867a8d6cdb8c9a4f0a8eec13012abbd8f3582f1de12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vendor=Red Hat, Inc.) 2024-05-20T22:25:33.844 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:33 smithi078 podman[52762]: 2024-05-20 22:25:33.595626473 +0000 UTC m=+0.582490936 container cleanup e4bbcbbe002698380a4c0867a8d6cdb8c9a4f0a8eec13012abbd8f3582f1de12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.expose-services=, version=8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True) 2024-05-20T22:25:33.845 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:33 smithi078 bash[52762]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078 2024-05-20T22:25:34.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 136641 11.38 MB/sec execute 160 sec latency 51.149 ms 2024-05-20T22:25:34.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:33 smithi078 podman[52774]: 2024-05-20 22:25:33.847276178 +0000 UTC m=+0.658158144 container remove e4bbcbbe002698380a4c0867a8d6cdb8c9a4f0a8eec13012abbd8f3582f1de12 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, release=754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, maintainer=Guillaume Abrioux , distribution-scope=public, ceph=True) 2024-05-20T22:25:34.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi078.service: Deactivated successfully. 2024-05-20T22:25:34.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 systemd[1]: Stopped Ceph mon.smithi078 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:25:34.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi078.service: Consumed 6.633s CPU time. 2024-05-20T22:25:34.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 systemd[1]: Starting Ceph mon.smithi078 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:25:34.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 podman[52901]: 2024-05-20 22:25:34.363636814 +0000 UTC m=+0.018542054 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:25:34.479 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 podman[52901]: 2024-05-20 22:25:34.482220154 +0000 UTC m=+0.137125375 container create ec5bf42c857c6a58e5efe5c4bb784cf7469e3b762ef84584c36ecfa6fa56d6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-20T22:25:35.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 137209 11.39 MB/sec execute 161 sec latency 37.582 ms 2024-05-20T22:25:35.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 podman[52901]: 2024-05-20 22:25:34.768844015 +0000 UTC m=+0.423749245 container init ec5bf42c857c6a58e5efe5c4bb784cf7469e3b762ef84584c36ecfa6fa56d6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-05-20T22:25:35.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 podman[52901]: 2024-05-20 22:25:34.772718986 +0000 UTC m=+0.427624212 container start ec5bf42c857c6a58e5efe5c4bb784cf7469e3b762ef84584c36ecfa6fa56d6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:25:35.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: set uid:gid to 167:167 (ceph:ceph) 2024-05-20T22:25:35.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev), process ceph-mon, pid 2 2024-05-20T22:25:35.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: pidfile_write: ignore empty --pid-file 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: load: jerasure load: lrc 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: RocksDB version: 7.9.2 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Git sha 0 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Compile date 2024-05-17 12:39:40 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: DB SUMMARY 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: DB Session ID: JKJKSQ2VWL7AA9SJ3RQO 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: CURRENT file: CURRENT 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: IDENTITY file: IDENTITY 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi078/store.db dir, Total Num: 1, files: 000024.sst 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi078/store.db: 000022.log size: 4875368 ; 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.error_if_exists: 0 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.create_if_missing: 0 2024-05-20T22:25:35.149 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.paranoid_checks: 1 2024-05-20T22:25:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.flush_verify_memtable_count: 1 2024-05-20T22:25:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-05-20T22:25:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-05-20T22:25:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.env: 0x55b810140c60 2024-05-20T22:25:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.fs: PosixFileSystem 2024-05-20T22:25:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.info_log: 0x55b81217f980 2024-05-20T22:25:35.150 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_file_opening_threads: 16 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.statistics: (nil) 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.use_fsync: 0 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_log_file_size: 0 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.log_file_time_to_roll: 0 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.keep_log_file_num: 1000 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.recycle_log_file_num: 0 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.allow_fallocate: 1 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.allow_mmap_reads: 0 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.allow_mmap_writes: 0 2024-05-20T22:25:35.151 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.use_direct_reads: 0 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.create_missing_column_families: 0 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.db_log_dir: 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.wal_dir: 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.table_cache_numshardbits: 6 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.WAL_ttl_seconds: 0 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.WAL_size_limit_MB: 0 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.is_fd_close_on_exec: 1 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.advise_random_on_open: 1 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.db_write_buffer_size: 0 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.write_buffer_manager: 0x55b8121521e0 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-05-20T22:25:35.152 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.use_adaptive_mutex: 0 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.rate_limiter: (nil) 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.wal_recovery_mode: 2 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.enable_thread_tracking: 0 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.enable_pipelined_write: 0 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.unordered_write: 0 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.row_cache: None 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.wal_filter: None 2024-05-20T22:25:35.153 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.allow_ingest_behind: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.two_write_queues: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.manual_wal_flush: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.wal_compression: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.atomic_flush: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.persist_stats_to_disk: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.write_dbid_to_manifest: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.log_readahead_size: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.best_efforts_recovery: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.allow_data_in_errors: 0 2024-05-20T22:25:35.154 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.db_host_id: __hostname__ 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.enforce_single_del_contracts: true 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_background_jobs: 2 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_background_compactions: -1 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_subcompactions: 1 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.delayed_write_rate : 16777216 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_total_wal_size: 0 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.stats_dump_period_sec: 600 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.stats_persist_period_sec: 600 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_open_files: -1 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bytes_per_sync: 0 2024-05-20T22:25:35.155 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.wal_bytes_per_sync: 0 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.strict_bytes_per_sync: 0 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_readahead_size: 0 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_background_flushes: -1 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Compression algorithms supported: 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kZSTD supported: 0 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kXpressCompression supported: 0 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kLZ4HCCompression supported: 1 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kZlibCompression supported: 1 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kSnappyCompression supported: 1 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kLZ4Compression supported: 1 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: kBZip2Compression supported: 0 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Fast CRC32 supported: Supported on x86 2024-05-20T22:25:35.156 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: DMutex implementation: pthread_mutex_t 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.merge_operator: 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_filter: None 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_filter_factory: None 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.sst_partitioner_factory: None 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.memtable_factory: SkipListFactory 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.table_factory: BlockBasedTable 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b81217fa80) 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks: 1 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: pin_top_level_index_and_filter: 1 2024-05-20T22:25:35.157 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_type: 0 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_index_type: 0 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_shortening: 1 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_block_hash_table_util_ratio: 0.750000 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: checksum: 4 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: no_block_cache: 0 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache: 0x55b812183090 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_name: BinnedLRUCache 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_options: 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: capacity : 536870912 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_shard_bits : 4 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: strict_capacity_limit : 0 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: high_pri_pool_ratio: 0.000 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_cache_compressed: (nil) 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: persistent_cache: (nil) 2024-05-20T22:25:35.158 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size: 4096 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_size_deviation: 10 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_restart_interval: 16 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: index_block_restart_interval: 1 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_block_size: 4096 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: partition_filters: 0 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: use_delta_encoding: 1 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: filter_policy: bloomfilter 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: whole_key_filtering: 1 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: verify_compression: 0 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: read_amp_bytes_per_bit: 0 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: format_version: 5 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_index_compression: 1 2024-05-20T22:25:35.159 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: block_align: 0 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_auto_readahead_size: 262144 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: prepopulate_block_cache: 0 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: initial_auto_readahead_size: 8192 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: num_file_reads_for_auto_readahead: 2 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.write_buffer_size: 33554432 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_write_buffer_number: 2 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression: NoCompression 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression: Disabled 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.prefix_extractor: nullptr 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.num_levels: 7 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-05-20T22:25:35.160 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.window_bits: -14 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.level: 32767 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.strategy: 0 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-05-20T22:25:35.161 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.enabled: false 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.target_file_size_base: 67108864 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.target_file_size_multiplier: 1 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-05-20T22:25:35.162 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-05-20T22:25:35.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-05-20T22:25:35.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-05-20T22:25:35.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-05-20T22:25:35.163 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.arena_block_size: 1048576 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.disable_auto_compactions: 0 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-05-20T22:25:35.164 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.table_properties_collectors: 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.inplace_update_support: 0 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.inplace_update_num_locks: 10000 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.memtable_huge_page_size: 0 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.bloom_locality: 0 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.max_successive_merges: 0 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.optimize_filters_for_hits: 0 2024-05-20T22:25:35.165 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.paranoid_file_checks: 0 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.force_consistency_checks: 1 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.report_bg_io_stats: 0 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.ttl: 2592000 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.periodic_compaction_seconds: 0 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.enable_blob_files: false 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.min_blob_size: 0 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.blob_file_size: 268435456 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.blob_compression_type: NoCompression 2024-05-20T22:25:35.166 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.enable_blob_garbage_collection: false 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.blob_file_starting_level: 0 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi078/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12206, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 85be3911-958d-4b3a-b8af-a893891b4437 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243934803899, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-05-20T22:25:35.167 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243934816303, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3021645, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12211, "largest_seqno": 13498, "table_properties": {"data_size": 3015690, "index_size": 3592, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 12156, "raw_average_key_size": 23, "raw_value_size": 3004300, "raw_average_value_size": 5788, "num_data_blocks": 164, "num_entries": 519, "num_filter_entries": 519, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1716243934, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "85be3911-958d-4b3a-b8af-a893891b4437", "db_session_id": "JKJKSQ2VWL7AA9SJ3RQO", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: EVENT_LOG_v1 {"time_micros": 1716243934816398, "job": 1, "event": "recovery_finished"} 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi078/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b812274000 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: DB pointer 0x55b81225e000 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** DB Stats ** 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:25:35.168 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L0 1/0 2.88 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 574.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: L6 1/0 7.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Sum 2/0 10.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 574.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 574.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** Compaction Stats [default] ** 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-05-20T22:25:35.169 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 574.0 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: AddFile(Keys): cumulative 0, interval 0 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Cumulative compaction: 0.00 GB write, 146.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Interval compaction: 0.00 GB write, 146.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache BinnedLRUCache@0x55b812183090#2 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-05-20T22:25:35.170 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: ** File Read Latency Histogram By Level [default] ** 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: starting mon.smithi078 rank 1 at public addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] at bind addrs [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi078 fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???) e2 preinit fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???).mds e7 new map 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???).mds e7 print_map 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: e7 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: legacy client fscid: 1 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Filesystem 'foofs' (1) 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: fs_name foofs 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: epoch 5 2024-05-20T22:25:35.171 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: created 2024-05-20T22:20:16.358680+0000 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: modified 2024-05-20T22:20:21.384193+0000 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: tableserver 0 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: root 0 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_timeout 60 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: session_autoclose 300 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_file_size 1099511627776 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_xattr_size 65536 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: required_client_features {} 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure 0 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: last_failure_osd_epoch 0 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: max_mds 1 2024-05-20T22:25:35.172 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: in 0 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: up {0=14476} 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: failed 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: damaged 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: stopped 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: data_pools [3] 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: metadata_pool 2 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: inline_data disabled 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: balancer 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: bal_rank_mask -1 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: standby_count_wanted 1 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: qdb_cluster leader: 0 members: 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi006.pburlb{0:14476} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.6:6834/3827922570,v1:172.21.15.6:6835/3827922570] compat {c=[1],r=[1],i=[7ff]}] 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.173 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: Standby daemons: 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout: [mds.foofs.smithi078.yocghn{-1:24311} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] compat {c=[1],r=[1],i=[7ff]}] 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 ceph-mon[52919]: mon.smithi078@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 bash[52901]: ec5bf42c857c6a58e5efe5c4bb784cf7469e3b762ef84584c36ecfa6fa56d6ac 2024-05-20T22:25:35.174 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:34 smithi078 systemd[1]: Started Ceph mon.smithi078 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:25:36.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 137892 11.46 MB/sec execute 162 sec latency 22.488 ms 2024-05-20T22:25:37.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 138560 11.54 MB/sec execute 163 sec latency 34.104 ms 2024-05-20T22:25:37.334 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 401 op/s 2024-05-20T22:25:37.334 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: 1: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-05-20T22:25:37.334 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:25:37.334 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:25:37.334 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: mgrmap e41: smithi006.ssyogp(active, since 20s), standbys: smithi078.vxzcbt 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: overall HEALTH_OK 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: mon.smithi078 calling monitor election 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: mon.smithi006 calling monitor election 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: mon.smithi006 is new leader, mons smithi006,smithi078 in quorum (ranks 0,1) 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: monmap epoch 3 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: last_changed 2024-05-20T22:25:36.244973+0000 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: created 2024-05-20T22:12:48.663636+0000 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: min_mon_release 19 (squid) 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: election_strategy: 1 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-05-20T22:25:37.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: 1: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-05-20T22:25:37.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:25:37.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:25:37.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: mgrmap e41: smithi006.ssyogp(active, since 20s), standbys: smithi078.vxzcbt 2024-05-20T22:25:37.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: overall HEALTH_OK 2024-05-20T22:25:37.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:37.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:37.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 401 op/s 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: 1: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: mgrmap e41: smithi006.ssyogp(active, since 20s), standbys: smithi078.vxzcbt 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: overall HEALTH_OK 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: mon.smithi078 calling monitor election 2024-05-20T22:25:37.356 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: mon.smithi006 calling monitor election 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi006"}]: dispatch 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mon metadata", "id": "smithi078"}]: dispatch 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: mon.smithi006 is new leader, mons smithi006,smithi078 in quorum (ranks 0,1) 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: monmap epoch 3 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: fsid 00e42188-16f6-11ef-bc9a-c7b262605968 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: last_changed 2024-05-20T22:25:36.244973+0000 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: created 2024-05-20T22:12:48.663636+0000 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: min_mon_release 19 (squid) 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: election_strategy: 1 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: 0: [v2:172.21.15.6:3300/0,v1:172.21.15.6:6789/0] mon.smithi006 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: 1: [v2:172.21.15.78:3300/0,v1:172.21.15.78:6789/0] mon.smithi078 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: osdmap e72: 8 total, 8 up, 8 in 2024-05-20T22:25:37.357 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: mgrmap e41: smithi006.ssyogp(active, since 20s), standbys: smithi078.vxzcbt 2024-05-20T22:25:37.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: overall HEALTH_OK 2024-05-20T22:25:37.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:37.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:37.358 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:25:38.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 139217 11.56 MB/sec execute 164 sec latency 21.195 ms 2024-05-20T22:25:39.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 139874 11.57 MB/sec execute 165 sec latency 23.864 ms 2024-05-20T22:25:39.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:39 smithi006 ceph-mon[65753]: pgmap v14: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 692 op/s 2024-05-20T22:25:39.434 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:39 smithi078 ceph-mon[52919]: pgmap v14: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 692 op/s 2024-05-20T22:25:40.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 140568 11.61 MB/sec execute 166 sec latency 32.737 ms 2024-05-20T22:25:41.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 141303 11.66 MB/sec execute 167 sec latency 30.277 ms 2024-05-20T22:25:41.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:41 smithi006 ceph-mon[65753]: pgmap v15: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-05-20T22:25:41.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:41 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:41.371 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:41 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:41.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:41 smithi078 ceph-mon[52919]: pgmap v15: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-05-20T22:25:41.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:41 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:41.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:41 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:42.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 141754 11.70 MB/sec execute 168 sec latency 30.146 ms 2024-05-20T22:25:42.894 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:42 smithi006 ceph-mon[65753]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-05-20T22:25:42.894 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:42.894 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:43.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 142234 11.71 MB/sec execute 169 sec latency 37.813 ms 2024-05-20T22:25:43.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:42 smithi078 ceph-mon[52919]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 520 op/s 2024-05-20T22:25:43.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:43.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:44.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 142685 11.71 MB/sec execute 170 sec latency 35.362 ms 2024-05-20T22:25:44.917 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:44 smithi006 ceph-mon[65753]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.6 MiB/s wr, 912 op/s 2024-05-20T22:25:45.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 143115 11.67 MB/sec execute 171 sec latency 30.678 ms 2024-05-20T22:25:45.140 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:44 smithi078 ceph-mon[52919]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 9.6 MiB/s wr, 912 op/s 2024-05-20T22:25:45.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:25:46.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 143598 11.66 MB/sec execute 172 sec latency 39.021 ms 2024-05-20T22:25:46.143 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:25:47.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 144090 11.68 MB/sec execute 173 sec latency 53.931 ms 2024-05-20T22:25:47.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 683 op/s 2024-05-20T22:25:47.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:47.456 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:25:47.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:25:47.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:25:47.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:47.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 683 op/s 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:25:47.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:48.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 144671 11.72 MB/sec execute 174 sec latency 44.082 ms 2024-05-20T22:25:48.464 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:48 smithi006 ceph-mon[65753]: Reconfiguring mon.smithi006 (monmap changed)... 2024-05-20T22:25:48.464 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:48 smithi006 ceph-mon[65753]: Reconfiguring daemon mon.smithi006 on smithi006 2024-05-20T22:25:48.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:48 smithi078 ceph-mon[52919]: Reconfiguring mon.smithi006 (monmap changed)... 2024-05-20T22:25:48.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:48 smithi078 ceph-mon[52919]: Reconfiguring daemon mon.smithi006 on smithi006 2024-05-20T22:25:49.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 145172 11.77 MB/sec execute 175 sec latency 38.128 ms 2024-05-20T22:25:49.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:49 smithi006 ceph-mon[65753]: pgmap v19: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 962 op/s 2024-05-20T22:25:49.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:49 smithi078 ceph-mon[52919]: pgmap v19: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 962 op/s 2024-05-20T22:25:49.701 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:25:50.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 145494 11.75 MB/sec execute 176 sec latency 77.517 ms 2024-05-20T22:25:50.593 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 671 op/s 2024-05-20T22:25:50.593 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:50.593 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:50.593 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: Reconfiguring mgr.smithi006.ssyogp (monmap changed)... 2024-05-20T22:25:50.593 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:25:50.593 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:25:50.593 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:50.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: Reconfiguring daemon mgr.smithi006.ssyogp on smithi006 2024-05-20T22:25:50.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:50.594 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:50 smithi006 ceph-mon[65753]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (2m) 25s ago 11m 15.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (11m) 25s ago 11m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (10m) 10s ago 10m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (2m) 25s ago 11m 43.8M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (5m) 25s ago 5m 36.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (5m) 10s ago 5m 12.8M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (2m) 25s ago 12m 480M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (65s) 10s ago 10m 450M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (45s) 25s ago 12m 31.6M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:25:50.752 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (16s) 10s ago 10m 22.2M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (5m) 25s ago 5m 278M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (5m) 10s ago 5m 87.4M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (2m) 25s ago 11m 11.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 10s ago 10m 19.0M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (9m) 25s ago 9m 480M 1937M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (9m) 25s ago 9m 400M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (8m) 25s ago 8m 280M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (8m) 25s ago 8m 439M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (8m) 10s ago 8m 484M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (7m) 10s ago 7m 530M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (7m) 10s ago 7m 446M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (6m) 10s ago 6m 371M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:25:50.753 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (72s) 25s ago 10m 56.9M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:25:50.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: pgmap v20: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 671 op/s 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: Reconfiguring mgr.smithi006.ssyogp (monmap changed)... 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi006.ssyogp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: Reconfiguring daemon mgr.smithi006.ssyogp on smithi006 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:50.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:50 smithi078 ceph-mon[52919]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:51.059 INFO:teuthology.orchestra.run.smithi006.stdout: 5 145787 11.73 MB/sec execute 177 sec latency 45.974 ms 2024-05-20T22:25:51.383 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:25:51.383 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:25:51.383 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:25:51.383 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:51.383 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 4 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:25:51.384 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:25:51.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:51 smithi006 ceph-mon[65753]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:51.614 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:51 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2937062925' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:25:51.882 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:25:51.882 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:25:51.882 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:25:51.882 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:25:51.882 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:25:51.882 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:25:51.883 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:25:51.883 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:25:51.883 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "4/23 daemons upgraded", 2024-05-20T22:25:51.883 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mon daemons", 2024-05-20T22:25:51.883 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:25:51.883 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:25:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:51 smithi078 ceph-mon[52919]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:51 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2937062925' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:25:52.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 146064 11.70 MB/sec execute 178 sec latency 63.381 ms 2024-05-20T22:25:52.481 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:25:53.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 146394 11.67 MB/sec execute 179 sec latency 41.918 ms 2024-05-20T22:25:53.379 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 671 op/s 2024-05-20T22:25:53.379 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:53.379 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:53.379 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:53.379 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: Reconfiguring crash.smithi006 (monmap changed)... 2024-05-20T22:25:53.380 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:25:53.380 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:53.380 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: Reconfiguring daemon crash.smithi006 on smithi006 2024-05-20T22:25:53.380 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:53 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1841182490' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:25:53.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 671 op/s 2024-05-20T22:25:53.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:25:53.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:53.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:53.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: Reconfiguring crash.smithi006 (monmap changed)... 2024-05-20T22:25:53.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:25:53.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:53.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: Reconfiguring daemon crash.smithi006 on smithi006 2024-05-20T22:25:53.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:53 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1841182490' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:25:54.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 146888 11.66 MB/sec execute 180 sec latency 32.139 ms 2024-05-20T22:25:55.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 147377 11.64 MB/sec execute 181 sec latency 42.165 ms 2024-05-20T22:25:55.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:55 smithi006 ceph-mon[65753]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 915 op/s 2024-05-20T22:25:55.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:55.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:55.397 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:25:55.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:55.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:55 smithi078 ceph-mon[52919]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 14 KiB/s rd, 9.6 MiB/s wr, 915 op/s 2024-05-20T22:25:55.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:55.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:55.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:25:55.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:56.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 147858 11.65 MB/sec execute 182 sec latency 37.953 ms 2024-05-20T22:25:56.418 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:56 smithi006 ceph-mon[65753]: Reconfiguring osd.0 (monmap changed)... 2024-05-20T22:25:56.418 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:56 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.0 on smithi006 2024-05-20T22:25:56.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:56 smithi078 ceph-mon[52919]: Reconfiguring osd.0 (monmap changed)... 2024-05-20T22:25:56.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:56 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.0 on smithi006 2024-05-20T22:25:57.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 148391 11.68 MB/sec execute 183 sec latency 39.276 ms 2024-05-20T22:25:57.198 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:57 smithi006 ceph-mon[65753]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.6 MiB/s wr, 523 op/s 2024-05-20T22:25:57.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:57 smithi078 ceph-mon[52919]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.6 MiB/s wr, 523 op/s 2024-05-20T22:25:58.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 148863 11.72 MB/sec execute 184 sec latency 39.610 ms 2024-05-20T22:25:59.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 149164 11.72 MB/sec execute 185 sec latency 47.092 ms 2024-05-20T22:25:59.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:59 smithi006 ceph-mon[65753]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 790 op/s 2024-05-20T22:25:59.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:59.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:59.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:25:59.244 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:25:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:25:59.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:59 smithi078 ceph-mon[52919]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 790 op/s 2024-05-20T22:25:59.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:59.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:25:59.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:25:59.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:25:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:00.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 149469 11.70 MB/sec execute 186 sec latency 43.546 ms 2024-05-20T22:26:00.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:00 smithi006 ceph-mon[65753]: Reconfiguring osd.1 (monmap changed)... 2024-05-20T22:26:00.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:00 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.1 on smithi006 2024-05-20T22:26:00.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:00 smithi078 ceph-mon[52919]: Reconfiguring osd.1 (monmap changed)... 2024-05-20T22:26:00.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:00 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.1 on smithi006 2024-05-20T22:26:01.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 149723 11.65 MB/sec execute 187 sec latency 60.228 ms 2024-05-20T22:26:01.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:01 smithi006 ceph-mon[65753]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-05-20T22:26:01.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:01.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:01.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:01.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:26:01.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:01.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:01 smithi078 ceph-mon[52919]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.2 MiB/s wr, 511 op/s 2024-05-20T22:26:01.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:01.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:01.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:01.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:26:01.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:02.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 149986 11.63 MB/sec execute 188 sec latency 38.130 ms 2024-05-20T22:26:02.337 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:02 smithi006 ceph-mon[65753]: Reconfiguring osd.2 (monmap changed)... 2024-05-20T22:26:02.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:02 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.2 on smithi006 2024-05-20T22:26:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:02 smithi078 ceph-mon[52919]: Reconfiguring osd.2 (monmap changed)... 2024-05-20T22:26:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:02 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.2 on smithi006 2024-05-20T22:26:03.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 150402 11.61 MB/sec execute 189 sec latency 39.990 ms 2024-05-20T22:26:03.291 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:03 smithi006 ceph-mon[65753]: pgmap v26: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 589 op/s 2024-05-20T22:26:03.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:03 smithi078 ceph-mon[52919]: pgmap v26: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 589 op/s 2024-05-20T22:26:04.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 150802 11.59 MB/sec execute 190 sec latency 38.140 ms 2024-05-20T22:26:05.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 151219 11.59 MB/sec execute 191 sec latency 39.654 ms 2024-05-20T22:26:05.161 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:04 smithi006 ceph-mon[65753]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 729 op/s 2024-05-20T22:26:05.161 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:04 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:05.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:04 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:05.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:04 smithi006 ceph-mon[65753]: Reconfiguring osd.3 (monmap changed)... 2024-05-20T22:26:05.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:04 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:26:05.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:04 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:05.162 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:04 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.3 on smithi006 2024-05-20T22:26:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:04 smithi078 ceph-mon[52919]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 729 op/s 2024-05-20T22:26:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:04 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:04 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:05.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:04 smithi078 ceph-mon[52919]: Reconfiguring osd.3 (monmap changed)... 2024-05-20T22:26:05.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:04 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:26:05.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:04 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:05.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:04 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.3 on smithi006 2024-05-20T22:26:06.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 151747 11.61 MB/sec execute 192 sec latency 35.024 ms 2024-05-20T22:26:06.965 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:06 smithi006 ceph-mon[65753]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 485 op/s 2024-05-20T22:26:06.966 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:06 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:06.966 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:06 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:06.966 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:06 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:26:06.966 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:06 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:07.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 152175 11.62 MB/sec execute 193 sec latency 39.924 ms 2024-05-20T22:26:07.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:06 smithi078 ceph-mon[52919]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 485 op/s 2024-05-20T22:26:07.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:06 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:07.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:06 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:07.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:06 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:26:07.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:06 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:07.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:07 smithi006 ceph-mon[65753]: Reconfiguring mds.foofs.smithi006.pburlb (monmap changed)... 2024-05-20T22:26:07.971 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:07 smithi006 ceph-mon[65753]: Reconfiguring daemon mds.foofs.smithi006.pburlb on smithi006 2024-05-20T22:26:08.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 152682 11.67 MB/sec execute 194 sec latency 24.520 ms 2024-05-20T22:26:08.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:07 smithi078 ceph-mon[52919]: Reconfiguring mds.foofs.smithi006.pburlb (monmap changed)... 2024-05-20T22:26:08.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:07 smithi078 ceph-mon[52919]: Reconfiguring daemon mds.foofs.smithi006.pburlb on smithi006 2024-05-20T22:26:08.976 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:08 smithi006 ceph-mon[65753]: pgmap v29: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 734 op/s 2024-05-20T22:26:09.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 153107 11.65 MB/sec execute 195 sec latency 39.792 ms 2024-05-20T22:26:09.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:08 smithi078 ceph-mon[52919]: pgmap v29: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 734 op/s 2024-05-20T22:26:10.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 153435 11.63 MB/sec execute 196 sec latency 41.646 ms 2024-05-20T22:26:10.250 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:10.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:10.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:10 smithi006 ceph-mon[65753]: Reconfiguring crash.smithi078 (monmap changed)... 2024-05-20T22:26:10.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:26:10.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:10.251 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:10 smithi006 ceph-mon[65753]: Reconfiguring daemon crash.smithi078 on smithi078 2024-05-20T22:26:10.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:10.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:10.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:10 smithi078 ceph-mon[52919]: Reconfiguring crash.smithi078 (monmap changed)... 2024-05-20T22:26:10.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:26:10.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:10.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:10 smithi078 ceph-mon[52919]: Reconfiguring daemon crash.smithi078 on smithi078 2024-05-20T22:26:11.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 153791 11.59 MB/sec execute 197 sec latency 51.181 ms 2024-05-20T22:26:11.258 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:11 smithi006 ceph-mon[65753]: pgmap v30: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 467 op/s 2024-05-20T22:26:11.259 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:11.259 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:11.259 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:26:11.259 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:26:11.259 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:11.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:11 smithi078 ceph-mon[52919]: pgmap v30: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 467 op/s 2024-05-20T22:26:11.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:11.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:11.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi078.vxzcbt", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-05-20T22:26:11.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-05-20T22:26:11.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:12.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 154282 11.59 MB/sec execute 198 sec latency 30.253 ms 2024-05-20T22:26:12.264 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:12 smithi006 ceph-mon[65753]: Reconfiguring mgr.smithi078.vxzcbt (monmap changed)... 2024-05-20T22:26:12.264 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:12 smithi006 ceph-mon[65753]: Reconfiguring daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:26:12.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:12 smithi078 ceph-mon[52919]: Reconfiguring mgr.smithi078.vxzcbt (monmap changed)... 2024-05-20T22:26:12.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:12 smithi078 ceph-mon[52919]: Reconfiguring daemon mgr.smithi078.vxzcbt on smithi078 2024-05-20T22:26:13.060 INFO:teuthology.orchestra.run.smithi006.stdout: 5 154868 11.61 MB/sec execute 199 sec latency 36.124 ms 2024-05-20T22:26:13.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:13 smithi006 ceph-mon[65753]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 675 op/s 2024-05-20T22:26:13.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:13.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:13.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:26:13.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:26:13.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:13.605 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:13 smithi078 ceph-mon[52919]: pgmap v31: 97 pgs: 97 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 675 op/s 2024-05-20T22:26:13.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:13.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:13.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-05-20T22:26:13.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-05-20T22:26:13.606 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:14.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 155353 11.63 MB/sec execute 200 sec latency 32.593 ms 2024-05-20T22:26:14.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:14 smithi006 ceph-mon[65753]: Reconfiguring mon.smithi078 (monmap changed)... 2024-05-20T22:26:14.301 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:14 smithi006 ceph-mon[65753]: Reconfiguring daemon mon.smithi078 on smithi078 2024-05-20T22:26:14.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:14 smithi078 ceph-mon[52919]: Reconfiguring mon.smithi078 (monmap changed)... 2024-05-20T22:26:14.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:14 smithi078 ceph-mon[52919]: Reconfiguring daemon mon.smithi078 on smithi078 2024-05-20T22:26:15.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 155706 11.63 MB/sec execute 201 sec latency 89.939 ms 2024-05-20T22:26:15.334 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:15 smithi006 ceph-mon[65753]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.7 MiB/s wr, 648 op/s 2024-05-20T22:26:15.334 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:15.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:15.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:26:15.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:15.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:15 smithi078 ceph-mon[52919]: pgmap v32: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.7 MiB/s wr, 648 op/s 2024-05-20T22:26:15.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:15.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:15.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:26:15.636 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:16.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 156064 11.65 MB/sec execute 202 sec latency 40.594 ms 2024-05-20T22:26:16.353 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:16 smithi006 ceph-mon[65753]: Reconfiguring osd.4 (monmap changed)... 2024-05-20T22:26:16.354 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:16 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.4 on smithi078 2024-05-20T22:26:16.354 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:16 smithi078 ceph-mon[52919]: Reconfiguring osd.4 (monmap changed)... 2024-05-20T22:26:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:16 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.4 on smithi078 2024-05-20T22:26:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:17.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 156354 11.63 MB/sec execute 203 sec latency 39.674 ms 2024-05-20T22:26:17.368 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:17 smithi006 ceph-mon[65753]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 507 op/s 2024-05-20T22:26:17.368 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:17.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:17.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:26:17.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:17.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:17 smithi078 ceph-mon[52919]: pgmap v33: 97 pgs: 97 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 507 op/s 2024-05-20T22:26:17.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:17.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:17.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:26:17.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:18.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 156715 11.62 MB/sec execute 204 sec latency 44.786 ms 2024-05-20T22:26:18.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:18 smithi006 ceph-mon[65753]: Reconfiguring osd.5 (monmap changed)... 2024-05-20T22:26:18.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:18 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.5 on smithi078 2024-05-20T22:26:18.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:18 smithi078 ceph-mon[52919]: Reconfiguring osd.5 (monmap changed)... 2024-05-20T22:26:18.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:18 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.5 on smithi078 2024-05-20T22:26:19.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 157147 11.60 MB/sec execute 205 sec latency 27.226 ms 2024-05-20T22:26:19.470 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:19 smithi006 ceph-mon[65753]: pgmap v34: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 763 op/s 2024-05-20T22:26:19.471 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:19 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:19.471 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:19 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:19.471 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:19 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:26:19.471 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:19 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:19.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:19 smithi078 ceph-mon[52919]: pgmap v34: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 763 op/s 2024-05-20T22:26:19.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:19 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:19.631 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:19 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:19.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:19 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:26:19.632 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:19 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:20.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 157653 11.59 MB/sec execute 206 sec latency 34.444 ms 2024-05-20T22:26:20.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:20 smithi078 ceph-mon[52919]: Reconfiguring osd.6 (monmap changed)... 2024-05-20T22:26:20.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:20 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.6 on smithi078 2024-05-20T22:26:20.478 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:20 smithi006 ceph-mon[65753]: Reconfiguring osd.6 (monmap changed)... 2024-05-20T22:26:20.478 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:20 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.6 on smithi078 2024-05-20T22:26:21.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 158211 11.60 MB/sec execute 207 sec latency 38.003 ms 2024-05-20T22:26:21.491 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:21 smithi006 ceph-mon[65753]: pgmap v35: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 514 op/s 2024-05-20T22:26:21.491 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:21.491 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:21.491 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:26:21.492 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:21.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:21 smithi078 ceph-mon[52919]: pgmap v35: 97 pgs: 97 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 514 op/s 2024-05-20T22:26:21.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:21.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:21.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:26:21.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:22.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 158709 11.60 MB/sec execute 208 sec latency 46.258 ms 2024-05-20T22:26:22.511 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:22 smithi006 ceph-mon[65753]: Reconfiguring osd.7 (monmap changed)... 2024-05-20T22:26:22.512 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:22 smithi006 ceph-mon[65753]: Reconfiguring daemon osd.7 on smithi078 2024-05-20T22:26:22.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:22 smithi078 ceph-mon[52919]: Reconfiguring osd.7 (monmap changed)... 2024-05-20T22:26:22.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:22 smithi078 ceph-mon[52919]: Reconfiguring daemon osd.7 on smithi078 2024-05-20T22:26:23.030 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:26:23.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 159224 11.64 MB/sec execute 209 sec latency 44.751 ms 2024-05-20T22:26:23.520 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:23 smithi006 ceph-mon[65753]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 557 op/s 2024-05-20T22:26:23.520 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:23 smithi006 ceph-mon[65753]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:23.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:23 smithi078 ceph-mon[52919]: pgmap v36: 97 pgs: 97 active+clean; 333 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 557 op/s 2024-05-20T22:26:23.650 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:23 smithi078 ceph-mon[52919]: from='client.44121 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:24.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 159663 11.67 MB/sec execute 210 sec latency 31.337 ms 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (3m) 58s ago 12m 15.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (11m) 58s ago 11m 6409k - 17.2.0 e1d6a67b021e dcdf500aa9c0 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (11m) 43s ago 11m 6409k - 17.2.0 e1d6a67b021e 5d8b5aac4320 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (2m) 58s ago 11m 43.8M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (6m) 58s ago 6m 36.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (6m) 43s ago 6m 12.8M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (2m) 58s ago 13m 480M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (98s) 43s ago 11m 450M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (79s) 58s ago 13m 31.6M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:26:24.128 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (49s) 43s ago 11m 22.2M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (5m) 58s ago 5m 278M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (5m) 43s ago 5m 87.4M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (2m) 58s ago 11m 11.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 43s ago 10m 19.0M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (10m) 58s ago 10m 480M 1937M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (10m) 58s ago 9m 400M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (9m) 58s ago 9m 280M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (9m) 58s ago 9m 439M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (8m) 43s ago 8m 484M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (8m) 43s ago 8m 530M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (7m) 43s ago 7m 446M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (7m) 43s ago 7m 371M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:26:24.129 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (105s) 58s ago 10m 56.9M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:26:24.766 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:26:24.766 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:26:24.766 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:26:24.766 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:24.766 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:26:24.766 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 4 2024-05-20T22:26:24.767 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:26:24.768 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 586 op/s 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: Reconfiguring mds.foofs.smithi078.yocghn (monmap changed)... 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: Reconfiguring daemon mds.foofs.smithi078.yocghn on smithi078 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: from='client.34186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:24.782 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:24 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2564888721' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:24.814 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.6 MiB/s wr, 586 op/s 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: Reconfiguring mds.foofs.smithi078.yocghn (monmap changed)... 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: Reconfiguring daemon mds.foofs.smithi078.yocghn on smithi078 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: from='client.34186 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:24.815 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:24 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2564888721' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:25.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 159976 11.65 MB/sec execute 211 sec latency 35.015 ms 2024-05-20T22:26:25.318 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "4/23 daemons upgraded", 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mon daemons", 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:26:25.319 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:26:26.000 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:26:26.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 160258 11.62 MB/sec execute 212 sec latency 76.713 ms 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all mon 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"}]: dispatch 2024-05-20T22:26:26.823 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"}]': finished 2024-05-20T22:26:26.824 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]: dispatch 2024-05-20T22:26:26.824 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-05-20T22:26:26.824 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:26 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/4116613508' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:26:26.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:26.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 535 op/s 2024-05-20T22:26:26.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all mon 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"}]: dispatch 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi006"}]': finished 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]: dispatch 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi078"}]': finished 2024-05-20T22:26:26.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:26 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/4116613508' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:26:27.062 INFO:teuthology.orchestra.run.smithi006.stdout: 5 160535 11.60 MB/sec execute 213 sec latency 41.319 ms 2024-05-20T22:26:28.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 160812 11.56 MB/sec execute 214 sec latency 55.322 ms 2024-05-20T22:26:28.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:28 smithi078 ceph-mon[52919]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 770 op/s 2024-05-20T22:26:28.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:28 smithi006 ceph-mon[65753]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 770 op/s 2024-05-20T22:26:29.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 161124 11.55 MB/sec execute 215 sec latency 93.223 ms 2024-05-20T22:26:30.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 161429 11.52 MB/sec execute 216 sec latency 60.941 ms 2024-05-20T22:26:30.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:29 smithi006 ceph-mon[65753]: Upgrade: Updating crash.smithi006 (1/2) 2024-05-20T22:26:30.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:30.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:26:30.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:30.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:29 smithi006 ceph-mon[65753]: Deploying daemon crash.smithi006 on smithi006 2024-05-20T22:26:30.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:29 smithi078 ceph-mon[52919]: Upgrade: Updating crash.smithi006 (1/2) 2024-05-20T22:26:30.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:29 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:30.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:29 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi006", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:26:30.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:29 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:30.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:29 smithi078 ceph-mon[52919]: Deploying daemon crash.smithi006 on smithi006 2024-05-20T22:26:30.961 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:30 smithi006 ceph-mon[65753]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-05-20T22:26:30.961 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:31.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 161764 11.50 MB/sec execute 217 sec latency 109.666 ms 2024-05-20T22:26:31.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:30 smithi078 ceph-mon[52919]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 514 op/s 2024-05-20T22:26:31.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:32.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 162088 11.49 MB/sec execute 218 sec latency 37.618 ms 2024-05-20T22:26:33.006 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:32 smithi006 ceph-mon[65753]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 663 op/s 2024-05-20T22:26:33.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 162645 11.53 MB/sec execute 219 sec latency 39.249 ms 2024-05-20T22:26:33.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:32 smithi078 ceph-mon[52919]: pgmap v41: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 663 op/s 2024-05-20T22:26:34.061 INFO:teuthology.orchestra.run.smithi006.stdout: 5 163105 11.55 MB/sec execute 220 sec latency 32.996 ms 2024-05-20T22:26:35.030 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:34 smithi006 ceph-mon[65753]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 664 op/s 2024-05-20T22:26:35.031 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:34 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:35.031 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:34 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:35.063 INFO:teuthology.orchestra.run.smithi006.stdout: 5 163581 11.56 MB/sec execute 221 sec latency 37.370 ms 2024-05-20T22:26:35.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:34 smithi078 ceph-mon[52919]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 664 op/s 2024-05-20T22:26:35.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:34 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:35.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:34 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:36.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 164129 11.56 MB/sec execute 222 sec latency 34.692 ms 2024-05-20T22:26:37.053 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:36 smithi006 ceph-mon[65753]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-05-20T22:26:37.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 164766 11.56 MB/sec execute 223 sec latency 39.857 ms 2024-05-20T22:26:37.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:36 smithi078 ceph-mon[52919]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-05-20T22:26:38.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:38 smithi006 ceph-mon[65753]: Upgrade: Updating crash.smithi078 (2/2) 2024-05-20T22:26:38.058 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:38.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:26:38.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:38.059 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:38 smithi006 ceph-mon[65753]: Deploying daemon crash.smithi078 on smithi078 2024-05-20T22:26:38.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 165380 11.57 MB/sec execute 224 sec latency 28.642 ms 2024-05-20T22:26:38.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:38 smithi078 ceph-mon[52919]: Upgrade: Updating crash.smithi078 (2/2) 2024-05-20T22:26:38.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:38.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi078", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-05-20T22:26:38.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:38.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:38 smithi078 ceph-mon[52919]: Deploying daemon crash.smithi078 on smithi078 2024-05-20T22:26:39.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 165805 11.58 MB/sec execute 225 sec latency 47.260 ms 2024-05-20T22:26:39.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:39 smithi006 ceph-mon[65753]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.6 MiB/s wr, 727 op/s 2024-05-20T22:26:39.092 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:39 smithi078 ceph-mon[52919]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.6 MiB/s wr, 727 op/s 2024-05-20T22:26:40.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 166257 11.61 MB/sec execute 226 sec latency 38.589 ms 2024-05-20T22:26:41.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 166565 11.60 MB/sec execute 227 sec latency 53.773 ms 2024-05-20T22:26:41.098 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:41 smithi006 ceph-mon[65753]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 493 op/s 2024-05-20T22:26:41.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:41 smithi078 ceph-mon[52919]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 493 op/s 2024-05-20T22:26:42.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 166879 11.60 MB/sec execute 228 sec latency 56.437 ms 2024-05-20T22:26:42.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:42 smithi078 ceph-mon[52919]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 727 op/s 2024-05-20T22:26:42.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:42.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:42.825 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:42.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:42 smithi006 ceph-mon[65753]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 727 op/s 2024-05-20T22:26:42.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:42.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:42.865 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:43.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167125 11.57 MB/sec execute 229 sec latency 43.267 ms 2024-05-20T22:26:44.064 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167391 11.55 MB/sec execute 230 sec latency 81.757 ms 2024-05-20T22:26:44.660 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:44 smithi006 ceph-mon[65753]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.8 MiB/s wr, 619 op/s 2024-05-20T22:26:44.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:44 smithi078 ceph-mon[52919]: pgmap v47: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.8 MiB/s wr, 619 op/s 2024-05-20T22:26:45.069 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167643 11.52 MB/sec execute 231 sec latency 50.302 ms 2024-05-20T22:26:45.697 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:45.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:26:46.069 INFO:teuthology.orchestra.run.smithi006.stdout: 5 167867 11.49 MB/sec execute 232 sec latency 92.439 ms 2024-05-20T22:26:46.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:46 smithi006 ceph-mon[65753]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-05-20T22:26:46.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:46.825 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:46.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:46 smithi078 ceph-mon[52919]: pgmap v48: 97 pgs: 97 active+clean; 335 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.2 MiB/s wr, 575 op/s 2024-05-20T22:26:46.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:46.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:47.069 INFO:teuthology.orchestra.run.smithi006.stdout: 5 168172 11.47 MB/sec execute 233 sec latency 67.767 ms 2024-05-20T22:26:48.069 INFO:teuthology.orchestra.run.smithi006.stdout: 5 168424 11.43 MB/sec execute 234 sec latency 77.740 ms 2024-05-20T22:26:48.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:48 smithi078 ceph-mon[52919]: pgmap v49: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 746 op/s 2024-05-20T22:26:48.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:48 smithi078 ceph-mon[52919]: mgrmap e42: smithi006.ssyogp(active, since 92s), standbys: smithi078.vxzcbt 2024-05-20T22:26:48.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:48 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:48.705 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:48 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:48.876 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:48 smithi006 ceph-mon[65753]: pgmap v49: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 746 op/s 2024-05-20T22:26:48.876 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:48 smithi006 ceph-mon[65753]: mgrmap e42: smithi006.ssyogp(active, since 92s), standbys: smithi078.vxzcbt 2024-05-20T22:26:48.876 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:48 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:48.876 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:48 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:49.069 INFO:teuthology.orchestra.run.smithi006.stdout: 5 168733 11.42 MB/sec execute 235 sec latency 51.331 ms 2024-05-20T22:26:50.069 INFO:teuthology.orchestra.run.smithi006.stdout: 5 169071 11.41 MB/sec execute 236 sec latency 58.246 ms 2024-05-20T22:26:50.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:50 smithi006 ceph-mon[65753]: pgmap v50: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-05-20T22:26:50.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:50 smithi078 ceph-mon[52919]: pgmap v50: 97 pgs: 97 active+clean; 333 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.4 MiB/s wr, 446 op/s 2024-05-20T22:26:51.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 169405 11.41 MB/sec execute 237 sec latency 64.900 ms 2024-05-20T22:26:52.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 169708 11.40 MB/sec execute 238 sec latency 69.281 ms 2024-05-20T22:26:52.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:52 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:52.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:52 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:52.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:52 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:52.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:52 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:52.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:52 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:52.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:52 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:52.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:52 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:52.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:52 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:53.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 169971 11.40 MB/sec execute 239 sec latency 53.791 ms 2024-05-20T22:26:53.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:53 smithi078 ceph-mon[52919]: pgmap v51: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-05-20T22:26:53.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:53.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:53.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:53 smithi006 ceph-mon[65753]: pgmap v51: 97 pgs: 97 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 540 op/s 2024-05-20T22:26:53.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:53.280 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:54.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170276 11.40 MB/sec execute 240 sec latency 50.061 ms 2024-05-20T22:26:55.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170524 11.38 MB/sec execute 241 sec latency 47.336 ms 2024-05-20T22:26:55.338 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:55 smithi006 ceph-mon[65753]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 397 op/s 2024-05-20T22:26:55.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:55 smithi078 ceph-mon[52919]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 397 op/s 2024-05-20T22:26:56.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170746 11.36 MB/sec execute 242 sec latency 70.311 ms 2024-05-20T22:26:56.581 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:26:57.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 170958 11.33 MB/sec execute 243 sec latency 51.833 ms 2024-05-20T22:26:57.110 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:57 smithi006 ceph-mon[65753]: pgmap v53: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 356 op/s 2024-05-20T22:26:57.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:57 smithi078 ceph-mon[52919]: pgmap v53: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 356 op/s 2024-05-20T22:26:57.748 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:26:57.748 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (3m) 6s ago 12m 14.1M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:26:57.748 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (23s) 6s ago 12m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:26:57.748 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (16s) 11s ago 11m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:26:57.749 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (3m) 6s ago 12m 43.5M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:26:57.749 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (6m) 6s ago 6m 36.8M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:26:57.749 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (6m) 11s ago 6m 13.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:26:57.749 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (3m) 6s ago 13m 499M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:26:57.749 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (2m) 11s ago 11m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:26:57.749 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (112s) 6s ago 14m 42.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:26:57.750 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (83s) 11s ago 11m 35.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:26:57.750 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (6m) 6s ago 6m 287M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:26:57.750 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (6m) 11s ago 6m 87.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:26:57.750 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (3m) 6s ago 12m 11.8M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:26:57.750 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 11s ago 11m 18.4M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:26:57.750 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (11m) 6s ago 11m 332M 1937M 17.2.0 e1d6a67b021e ee68d261ab09 2024-05-20T22:26:57.750 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (10m) 6s ago 10m 493M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:26:57.751 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (10m) 6s ago 10m 373M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:26:57.751 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (9m) 6s ago 9m 518M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:26:57.751 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (9m) 11s ago 9m 545M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:26:57.751 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (8m) 11s ago 8m 356M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:26:57.751 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (8m) 11s ago 8m 543M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:26:57.751 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (8m) 11s ago 8m 483M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:26:57.751 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (2m) 6s ago 11m 65.1M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:26:58.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 171219 11.30 MB/sec execute 244 sec latency 66.427 ms 2024-05-20T22:26:58.127 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:58 smithi006 ceph-mon[65753]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:58.127 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:58 smithi006 ceph-mon[65753]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:58.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:58 smithi078 ceph-mon[52919]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:58.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:58 smithi078 ceph-mon[52919]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:58.503 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:26:58.503 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:26:58.503 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:26:58.503 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 4 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:26:58.504 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:26:59.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 171495 11.28 MB/sec execute 245 sec latency 69.736 ms 2024-05-20T22:26:59.105 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:26:59.105 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:26:59.105 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:26:59.105 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "6/23 daemons upgraded", 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:26:59.106 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:26:59.154 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2797849055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.155 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.156 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.156 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.156 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"}]: dispatch 2024-05-20T22:26:59.156 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"}]': finished 2024-05-20T22:26:59.156 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]: dispatch 2024-05-20T22:26:59.156 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-05-20T22:26:59.156 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:26:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:26:59.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: pgmap v54: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-05-20T22:26:59.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:26:59.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2797849055' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi006"}]': finished 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]: dispatch 2024-05-20T22:26:59.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi078"}]': finished 2024-05-20T22:26:59.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:26:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:26:59.830 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:27:00.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 171803 11.26 MB/sec execute 246 sec latency 45.726 ms 2024-05-20T22:27:00.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:00 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all crash 2024-05-20T22:27:00.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:00 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:27:00.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:00 smithi006 ceph-mon[65753]: Upgrade: osd.0 is safe to restart 2024-05-20T22:27:00.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:00 smithi006 ceph-mon[65753]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:00.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:00 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1287703791' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:27:00.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:00 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all crash 2024-05-20T22:27:00.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:00 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-05-20T22:27:00.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:00 smithi078 ceph-mon[52919]: Upgrade: osd.0 is safe to restart 2024-05-20T22:27:00.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:00 smithi078 ceph-mon[52919]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:00.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:00 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1287703791' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:27:01.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 172127 11.24 MB/sec execute 247 sec latency 53.422 ms 2024-05-20T22:27:01.191 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:01 smithi006 ceph-mon[65753]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 338 op/s 2024-05-20T22:27:01.191 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:01 smithi078 ceph-mon[52919]: pgmap v55: 97 pgs: 97 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.6 MiB/s wr, 338 op/s 2024-05-20T22:27:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:02.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 172449 11.22 MB/sec execute 248 sec latency 54.685 ms 2024-05-20T22:27:02.457 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:02 smithi006 ceph-mon[65753]: Upgrade: Updating osd.0 2024-05-20T22:27:02.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:02 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:02.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:02 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:27:02.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:02 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:02.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:02 smithi006 ceph-mon[65753]: Deploying daemon osd.0 on smithi006 2024-05-20T22:27:02.458 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:02 smithi006 ceph-mon[65753]: pgmap v56: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.4 MiB/s wr, 465 op/s 2024-05-20T22:27:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:02 smithi078 ceph-mon[52919]: Upgrade: Updating osd.0 2024-05-20T22:27:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:02 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:02 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-05-20T22:27:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:02 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:02 smithi078 ceph-mon[52919]: Deploying daemon osd.0 on smithi006 2024-05-20T22:27:02.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:02 smithi078 ceph-mon[52919]: pgmap v56: 97 pgs: 97 active+clean; 330 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.4 MiB/s wr, 465 op/s 2024-05-20T22:27:03.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 172800 11.23 MB/sec execute 249 sec latency 56.695 ms 2024-05-20T22:27:04.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 173172 11.23 MB/sec execute 250 sec latency 47.389 ms 2024-05-20T22:27:04.776 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:04 smithi006 ceph-mon[65753]: pgmap v57: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 435 op/s 2024-05-20T22:27:04.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:04 smithi078 ceph-mon[52919]: pgmap v57: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 435 op/s 2024-05-20T22:27:05.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 173502 11.24 MB/sec execute 251 sec latency 49.619 ms 2024-05-20T22:27:06.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 173828 11.24 MB/sec execute 252 sec latency 43.567 ms 2024-05-20T22:27:06.287 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:06 smithi006 systemd[1]: Stopping Ceph osd.0 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:27:06.538 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:06 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[34880]: 2024-05-20T22:27:06.536+0000 7fbc35373700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:27:06.539 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:06 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[34880]: 2024-05-20T22:27:06.536+0000 7fbc35373700 -1 osd.0 72 *** Got signal Terminated *** 2024-05-20T22:27:06.539 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:06 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[34880]: 2024-05-20T22:27:06.536+0000 7fbc35373700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:27:06.539 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:06 smithi006 ceph-mon[65753]: pgmap v58: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.4 MiB/s wr, 344 op/s 2024-05-20T22:27:06.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:06 smithi078 ceph-mon[52919]: pgmap v58: 97 pgs: 97 active+clean; 327 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.4 MiB/s wr, 344 op/s 2024-05-20T22:27:07.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174100 11.22 MB/sec execute 253 sec latency 51.431 ms 2024-05-20T22:27:07.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:07 smithi006 ceph-mon[65753]: osd.0 marked itself down and dead 2024-05-20T22:27:07.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:07 smithi078 ceph-mon[52919]: osd.0 marked itself down and dead 2024-05-20T22:27:08.070 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174225 11.19 MB/sec execute 254 sec latency 539.086 ms 2024-05-20T22:27:08.454 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:08 smithi006 podman[71418]: 2024-05-20 22:27:08.187200336 +0000 UTC m=+1.780652294 container died ee68d261ab091488d2459ad142ba624bc6fc0687f11096158e7c65f0ecf6240e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_CLEAN=True, name=centos-stream, GIT_BRANCH=HEAD, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, release=754) 2024-05-20T22:27:08.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:08 smithi006 ceph-mon[65753]: pgmap v59: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-05-20T22:27:08.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:08 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:27:08.705 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:08 smithi006 ceph-mon[65753]: osdmap e73: 8 total, 7 up, 8 in 2024-05-20T22:27:08.705 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:08 smithi006 podman[71418]: 2024-05-20 22:27:08.57291137 +0000 UTC m=+2.166363325 container cleanup ee68d261ab091488d2459ad142ba624bc6fc0687f11096158e7c65f0ecf6240e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, release=754, vcs-type=git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, version=8, ceph=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8) 2024-05-20T22:27:08.705 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:08 smithi006 bash[71418]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0 2024-05-20T22:27:08.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:08 smithi078 ceph-mon[52919]: pgmap v59: 97 pgs: 97 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.7 MiB/s wr, 531 op/s 2024-05-20T22:27:08.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:08 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:27:08.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:08 smithi078 ceph-mon[52919]: osdmap e73: 8 total, 7 up, 8 in 2024-05-20T22:27:08.956 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:08 smithi006 podman[71431]: 2024-05-20 22:27:08.953471378 +0000 UTC m=+0.757686102 container remove ee68d261ab091488d2459ad142ba624bc6fc0687f11096158e7c65f0ecf6240e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, version=8, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.tags=base centos centos-stream, name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-type=git, release=754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-05-20T22:27:09.071 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174456 11.16 MB/sec execute 255 sec latency 1063.408 ms 2024-05-20T22:27:09.715 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:09 smithi006 ceph-mon[65753]: osdmap e74: 8 total, 7 up, 8 in 2024-05-20T22:27:09.716 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:09 smithi006 podman[71547]: 2024-05-20 22:27:09.480727211 +0000 UTC m=+0.029198675 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:27:09.716 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:09 smithi006 podman[71547]: 2024-05-20 22:27:09.694253289 +0000 UTC m=+0.242724750 container create f22a7c3e7562cd9cf4f683db7f021f96f81138a1f04ace0a68c1d246a3984190 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-20T22:27:09.799 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:09 smithi078 ceph-mon[52919]: osdmap e74: 8 total, 7 up, 8 in 2024-05-20T22:27:10.071 INFO:teuthology.orchestra.run.smithi006.stdout: 5 174869 11.15 MB/sec execute 256 sec latency 57.361 ms 2024-05-20T22:27:10.469 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:10 smithi006 podman[71547]: 2024-05-20 22:27:10.282399253 +0000 UTC m=+0.830870734 container init f22a7c3e7562cd9cf4f683db7f021f96f81138a1f04ace0a68c1d246a3984190 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-20T22:27:10.470 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:10 smithi006 podman[71547]: 2024-05-20 22:27:10.292883914 +0000 UTC m=+0.841355373 container start f22a7c3e7562cd9cf4f683db7f021f96f81138a1f04ace0a68c1d246a3984190 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-05-20T22:27:10.470 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:10 smithi006 podman[71547]: 2024-05-20 22:27:10.466214646 +0000 UTC m=+1.014686103 container attach f22a7c3e7562cd9cf4f683db7f021f96f81138a1f04ace0a68c1d246a3984190 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:27:10.721 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:10 smithi006 ceph-mon[65753]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.5 MiB/s wr, 376 op/s 2024-05-20T22:27:10.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:10 smithi078 ceph-mon[52919]: pgmap v62: 97 pgs: 13 stale+active+clean, 84 active+clean; 339 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.5 MiB/s wr, 376 op/s 2024-05-20T22:27:10.980 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:10 smithi006 podman[71547]: 2024-05-20 22:27:10.772133202 +0000 UTC m=+1.320604667 container died f22a7c3e7562cd9cf4f683db7f021f96f81138a1f04ace0a68c1d246a3984190 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-20T22:27:11.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 175425 11.15 MB/sec execute 257 sec latency 47.924 ms 2024-05-20T22:27:11.732 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:11 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 101/1689 objects degraded (5.980%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:11.733 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:11 smithi006 podman[71571]: 2024-05-20 22:27:11.567800155 +0000 UTC m=+0.785923757 container remove f22a7c3e7562cd9cf4f683db7f021f96f81138a1f04ace0a68c1d246a3984190 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-05-20T22:27:11.733 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:11 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.0.service: Deactivated successfully. 2024-05-20T22:27:11.733 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:11 smithi006 systemd[1]: Stopped Ceph osd.0 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:27:11.733 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:11 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.0.service: Consumed 3min 4.293s CPU time. 2024-05-20T22:27:11.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:11 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 101/1689 objects degraded (5.980%), 11 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:11.983 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:11 smithi006 systemd[1]: Starting Ceph osd.0 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:27:12.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 175889 11.14 MB/sec execute 258 sec latency 43.744 ms 2024-05-20T22:27:12.240 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:12 smithi006 podman[71667]: 2024-05-20 22:27:12.145491924 +0000 UTC m=+0.025606424 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:27:12.498 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:12 smithi006 podman[71667]: 2024-05-20 22:27:12.284047484 +0000 UTC m=+0.164161964 container create b6846317e92299e99b33a521d46bab43632fd6c782c9dac3a2e87d31b294eb50 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514) 2024-05-20T22:27:12.753 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:12 smithi006 ceph-mon[65753]: pgmap v63: 97 pgs: 11 active+undersized+degraded, 3 active+undersized, 77 active+clean, 6 stale+active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 101/1689 objects degraded (5.980%) 2024-05-20T22:27:12.753 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:12 smithi006 podman[71667]: 2024-05-20 22:27:12.669265418 +0000 UTC m=+0.549379908 container init b6846317e92299e99b33a521d46bab43632fd6c782c9dac3a2e87d31b294eb50 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:27:12.753 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:12 smithi006 podman[71667]: 2024-05-20 22:27:12.680436768 +0000 UTC m=+0.560551254 container start b6846317e92299e99b33a521d46bab43632fd6c782c9dac3a2e87d31b294eb50 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-20T22:27:12.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:12 smithi078 ceph-mon[52919]: pgmap v63: 97 pgs: 11 active+undersized+degraded, 3 active+undersized, 77 active+clean, 6 stale+active+clean; 341 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 409 op/s; 101/1689 objects degraded (5.980%) 2024-05-20T22:27:13.015 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:12 smithi006 podman[71667]: 2024-05-20 22:27:12.842677074 +0000 UTC m=+0.722791552 container attach b6846317e92299e99b33a521d46bab43632fd6c782c9dac3a2e87d31b294eb50 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:27:13.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 176359 11.16 MB/sec execute 259 sec latency 35.427 ms 2024-05-20T22:27:13.526 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate[71681]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:27:13.526 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 bash[71667]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:27:13.526 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 bash[71667]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T22:27:13.526 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 bash[71667]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T22:27:13.527 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 bash[71667]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T22:27:13.527 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 bash[71667]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-20T22:27:13.527 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 bash[71667]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:27:13.527 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 bash[71667]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-20T22:27:13.527 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate[71681]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T22:27:13.528 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate[71681]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T22:27:13.528 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate[71681]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T22:27:13.528 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate[71681]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-05-20T22:27:13.528 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate[71681]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-05-20T22:27:13.528 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate[71681]: --> ceph-volume raw activate successful for osd ID: 0 2024-05-20T22:27:13.528 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:13 smithi006 podman[71667]: 2024-05-20 22:27:13.351594229 +0000 UTC m=+1.231708705 container died b6846317e92299e99b33a521d46bab43632fd6c782c9dac3a2e87d31b294eb50 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-05-20T22:27:14.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 176974 11.20 MB/sec execute 260 sec latency 30.279 ms 2024-05-20T22:27:14.536 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:14 smithi006 podman[71817]: 2024-05-20 22:27:14.34402383 +0000 UTC m=+0.984849842 container remove b6846317e92299e99b33a521d46bab43632fd6c782c9dac3a2e87d31b294eb50 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2) 2024-05-20T22:27:14.788 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:14 smithi006 ceph-mon[65753]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 606 op/s; 210/1719 objects degraded (12.216%) 2024-05-20T22:27:14.789 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:14 smithi006 podman[71876]: 2024-05-20 22:27:14.517149104 +0000 UTC m=+0.024670099 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:27:14.789 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:14 smithi006 podman[71876]: 2024-05-20 22:27:14.704284355 +0000 UTC m=+0.211805528 container create 4f65f81f560e80ca7c366b1937165cb07b661ecc5506cb94fbae39386bc9a315 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-05-20T22:27:14.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:14 smithi078 ceph-mon[52919]: pgmap v64: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.2 MiB/s wr, 606 op/s; 210/1719 objects degraded (12.216%) 2024-05-20T22:27:15.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 177448 11.24 MB/sec execute 261 sec latency 30.970 ms 2024-05-20T22:27:15.290 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:15 smithi006 podman[71876]: 2024-05-20 22:27:15.092295918 +0000 UTC m=+0.599816910 container init 4f65f81f560e80ca7c366b1937165cb07b661ecc5506cb94fbae39386bc9a315 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:27:15.290 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:15 smithi006 podman[71876]: 2024-05-20 22:27:15.097704371 +0000 UTC m=+0.605225362 container start 4f65f81f560e80ca7c366b1937165cb07b661ecc5506cb94fbae39386bc9a315 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:27:15.290 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:15 smithi006 ceph-osd[71894]: -- 172.21.15.6:0/979808985 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560c3de57680 con 0x560c3de20c00 2024-05-20T22:27:15.542 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:15 smithi006 bash[71876]: 4f65f81f560e80ca7c366b1937165cb07b661ecc5506cb94fbae39386bc9a315 2024-05-20T22:27:15.819 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:15 smithi006 systemd[1]: Started Ceph osd.0 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:27:15.819 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:15 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[71890]: 2024-05-20T22:27:15.670+0000 7f5225d4d7c0 -1 Falling back to public interface 2024-05-20T22:27:16.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 177871 11.22 MB/sec execute 262 sec latency 57.339 ms 2024-05-20T22:27:16.579 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:16 smithi006 ceph-mon[65753]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.8 MiB/s wr, 325 op/s; 210/1719 objects degraded (12.216%) 2024-05-20T22:27:16.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:16.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:16.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:16.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:16.580 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:16.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:16 smithi078 ceph-mon[52919]: pgmap v65: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.8 MiB/s wr, 325 op/s; 210/1719 objects degraded (12.216%) 2024-05-20T22:27:16.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:16.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:16.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:16.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:16.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:17.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 178165 11.20 MB/sec execute 263 sec latency 48.420 ms 2024-05-20T22:27:17.586 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:17 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 214/1749 objects degraded (12.236%), 29 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:17.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:17 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 214/1749 objects degraded (12.236%), 29 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:18.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 178569 11.19 MB/sec execute 264 sec latency 49.329 ms 2024-05-20T22:27:18.590 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:18 smithi006 ceph-mon[65753]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 599 op/s; 214/1749 objects degraded (12.236%) 2024-05-20T22:27:18.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:18 smithi078 ceph-mon[52919]: pgmap v66: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 599 op/s; 214/1749 objects degraded (12.236%) 2024-05-20T22:27:19.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 178987 11.18 MB/sec execute 265 sec latency 52.514 ms 2024-05-20T22:27:20.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 179366 11.16 MB/sec execute 266 sec latency 34.163 ms 2024-05-20T22:27:20.670 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:20 smithi006 ceph-mon[65753]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 544 op/s; 214/1749 objects degraded (12.236%) 2024-05-20T22:27:20.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:20 smithi078 ceph-mon[52919]: pgmap v67: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 344 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.5 MiB/s wr, 544 op/s; 214/1749 objects degraded (12.236%) 2024-05-20T22:27:21.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 179770 11.17 MB/sec execute 267 sec latency 45.809 ms 2024-05-20T22:27:22.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 180319 11.19 MB/sec execute 268 sec latency 33.043 ms 2024-05-20T22:27:22.705 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:22 smithi006 ceph-mon[65753]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 717 op/s; 204/1584 objects degraded (12.879%) 2024-05-20T22:27:22.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:22 smithi078 ceph-mon[52919]: pgmap v68: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 331 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 717 op/s; 204/1584 objects degraded (12.879%) 2024-05-20T22:27:23.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 180798 11.23 MB/sec execute 269 sec latency 30.564 ms 2024-05-20T22:27:23.469 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:23 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[71890]: 2024-05-20T22:27:23.320+0000 7f5225d4d7c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-05-20T22:27:23.720 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:23 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 204/1602 objects degraded (12.734%), 29 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:23 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 204/1602 objects degraded (12.734%), 29 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:24.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 181162 11.22 MB/sec execute 270 sec latency 45.214 ms 2024-05-20T22:27:24.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:24 smithi006 ceph-mon[65753]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 658 op/s; 204/1602 objects degraded (12.734%) 2024-05-20T22:27:24.778 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:24 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[71890]: 2024-05-20T22:27:24.750+0000 7f5225d4d7c0 -1 osd.0 72 log_to_monitors true 2024-05-20T22:27:24.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:24 smithi078 ceph-mon[52919]: pgmap v69: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.6 MiB/s wr, 658 op/s; 204/1602 objects degraded (12.734%) 2024-05-20T22:27:25.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 181555 11.21 MB/sec execute 271 sec latency 40.592 ms 2024-05-20T22:27:25.756 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:25 smithi006 ceph-mon[65753]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:27:26.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 181848 11.19 MB/sec execute 272 sec latency 151.801 ms 2024-05-20T22:27:26.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:25 smithi078 ceph-mon[52919]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-05-20T22:27:26.307 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:27:26 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[71890]: 2024-05-20T22:27:26.245+0000 7f521319c700 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:27:26.812 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:26 smithi006 ceph-mon[65753]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 526 op/s; 204/1602 objects degraded (12.734%) 2024-05-20T22:27:26.812 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:26 smithi006 ceph-mon[65753]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:27:26.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:26 smithi006 ceph-mon[65753]: osdmap e75: 8 total, 7 up, 8 in 2024-05-20T22:27:26.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:26 smithi006 ceph-mon[65753]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:27:26.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:26.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:26.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:26 smithi006 ceph-mon[65753]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' 2024-05-20T22:27:27.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182172 11.17 MB/sec execute 273 sec latency 116.887 ms 2024-05-20T22:27:27.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:26 smithi078 ceph-mon[52919]: pgmap v70: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 330 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 526 op/s; 204/1602 objects degraded (12.734%) 2024-05-20T22:27:27.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:26 smithi078 ceph-mon[52919]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-05-20T22:27:27.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:26 smithi078 ceph-mon[52919]: osdmap e75: 8 total, 7 up, 8 in 2024-05-20T22:27:27.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:26 smithi078 ceph-mon[52919]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:27:27.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:27.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:27.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:26 smithi078 ceph-mon[52919]: from='osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405]' entity='osd.0' 2024-05-20T22:27:27.824 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:27 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:27:27.824 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:27 smithi006 ceph-mon[65753]: osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405] boot 2024-05-20T22:27:27.824 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:27 smithi006 ceph-mon[65753]: osdmap e76: 8 total, 8 up, 8 in 2024-05-20T22:27:27.824 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:27:28.076 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182295 11.14 MB/sec execute 274 sec latency 802.396 ms 2024-05-20T22:27:28.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:27 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:27:28.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:27 smithi078 ceph-mon[52919]: osd.0 [v2:172.21.15.6:6802/524974405,v1:172.21.15.6:6803/524974405] boot 2024-05-20T22:27:28.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:27 smithi078 ceph-mon[52919]: osdmap e76: 8 total, 8 up, 8 in 2024-05-20T22:27:28.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:27 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-05-20T22:27:29.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:28 smithi006 ceph-mon[65753]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 749 op/s; 209/1638 objects degraded (12.759%) 2024-05-20T22:27:29.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:29.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:29.037 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:28 smithi006 ceph-mon[65753]: osdmap e77: 8 total, 8 up, 8 in 2024-05-20T22:27:29.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182295 11.10 MB/sec execute 275 sec latency 1802.426 ms 2024-05-20T22:27:29.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:28 smithi078 ceph-mon[52919]: pgmap v73: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 749 op/s; 209/1638 objects degraded (12.759%) 2024-05-20T22:27:29.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:29.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:29.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:28 smithi078 ceph-mon[52919]: osdmap e77: 8 total, 8 up, 8 in 2024-05-20T22:27:30.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182373 11.07 MB/sec execute 276 sec latency 2458.715 ms 2024-05-20T22:27:30.474 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:30 smithi006 ceph-mon[65753]: osdmap e78: 8 total, 8 up, 8 in 2024-05-20T22:27:30.499 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:27:30.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:30 smithi078 ceph-mon[52919]: osdmap e78: 8 total, 8 up, 8 in 2024-05-20T22:27:31.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182631 11.04 MB/sec execute 277 sec latency 682.377 ms 2024-05-20T22:27:31.510 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:31 smithi006 ceph-mon[65753]: pgmap v76: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 756 op/s; 209/1638 objects degraded (12.759%) 2024-05-20T22:27:31.510 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:31 smithi006 ceph-mon[65753]: osdmap e79: 8 total, 8 up, 8 in 2024-05-20T22:27:31.510 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:31.510 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:31.511 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:31 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 209/1638 objects degraded (12.759%), 29 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:31.584 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (4m) 5s ago 13m 14.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (57s) 5s ago 13m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (50s) 45s ago 12m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (4m) 5s ago 12m 44.0M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (7m) 5s ago 7m 36.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (7m) 45s ago 7m 13.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (3m) 5s ago 14m 502M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (2m) 45s ago 12m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (2m) 5s ago 14m 46.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (117s) 45s ago 12m 35.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:27:31.585 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (6m) 5s ago 6m 290M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (6m) 45s ago 6m 87.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (4m) 5s ago 12m 16.0M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (3m) 45s ago 12m 18.4M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (17s) 5s ago 11m 12.5M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (11m) 5s ago 11m 548M 1937M 17.2.0 e1d6a67b021e ac383466e6c4 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (10m) 5s ago 10m 403M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (10m) 5s ago 10m 543M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (9m) 45s ago 9m 545M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (9m) 45s ago 9m 356M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (9m) 45s ago 9m 543M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (8m) 45s ago 8m 483M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:27:31.586 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (2m) 5s ago 11m 67.5M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:27:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:31 smithi078 ceph-mon[52919]: pgmap v76: 97 pgs: 29 active+undersized+degraded, 7 active+undersized, 61 active+clean; 334 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.6 MiB/s wr, 756 op/s; 209/1638 objects degraded (12.759%) 2024-05-20T22:27:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:31 smithi078 ceph-mon[52919]: osdmap e79: 8 total, 8 up, 8 in 2024-05-20T22:27:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:31 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 209/1638 objects degraded (12.759%), 29 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:32.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 182940 11.03 MB/sec execute 278 sec latency 895.519 ms 2024-05-20T22:27:32.329 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:27:32.330 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 5 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:27:32.331 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:27:32.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:32 smithi006 ceph-mon[65753]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:32.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:32 smithi006 ceph-mon[65753]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:32.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:32 smithi006 ceph-mon[65753]: osdmap e80: 8 total, 8 up, 8 in 2024-05-20T22:27:32.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:32 smithi078 ceph-mon[52919]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:32.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:32 smithi078 ceph-mon[52919]: from='client.44167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:32.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:32 smithi078 ceph-mon[52919]: osdmap e80: 8 total, 8 up, 8 in 2024-05-20T22:27:32.976 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:27:32.976 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:27:32.976 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:27:32.976 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "7/23 daemons upgraded", 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:27:32.977 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:27:33.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 183175 11.00 MB/sec execute 279 sec latency 552.053 ms 2024-05-20T22:27:33.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: pgmap v79: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 18 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized, 73 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 280 KiB/s wr, 47 op/s; 190/1563 objects degraded (12.156%); 75 KiB/s, 8 objects/s recovering 2024-05-20T22:27:33.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:33.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: osdmap e81: 8 total, 8 up, 8 in 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2412479482' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN Degraded data redundancy: 190/1563 objects degraded (12.156%), 22 pgs degraded 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 190/1563 objects degraded (12.156%), 22 pgs degraded 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-05-20T22:27:33.583 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.3 is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.10 is active+recovery_wait+undersized+degraded+remapped, acting [6,5] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1b is active+recovering+undersized+degraded+remapped, acting [4,7] 2024-05-20T22:27:33.584 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1f is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-05-20T22:27:33.585 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-05-20T22:27:33.585 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: pgmap v79: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 18 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized, 73 active+clean; 326 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 280 KiB/s wr, 47 op/s; 190/1563 objects degraded (12.156%); 75 KiB/s, 8 objects/s recovering 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: osdmap e81: 8 total, 8 up, 8 in 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:27:33.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:33.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2412479482' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:27:33.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:27:33.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:34.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 183633 11.02 MB/sec execute 280 sec latency 786.860 ms 2024-05-20T22:27:34.320 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:34 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:34.320 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:34 smithi006 ceph-mon[65753]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-05-20T22:27:34.320 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:34 smithi006 ceph-mon[65753]: from='client.44179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:34.320 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:34 smithi006 ceph-mon[65753]: osdmap e82: 8 total, 8 up, 8 in 2024-05-20T22:27:34.320 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:34 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3820611529' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:27:34.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:34 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:34.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:34 smithi078 ceph-mon[52919]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-05-20T22:27:34.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:34 smithi078 ceph-mon[52919]: from='client.44179 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:27:34.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:34 smithi078 ceph-mon[52919]: osdmap e82: 8 total, 8 up, 8 in 2024-05-20T22:27:34.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:34 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3820611529' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:27:35.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 184096 11.05 MB/sec execute 281 sec latency 1002.944 ms 2024-05-20T22:27:35.330 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:35 smithi006 ceph-mon[65753]: pgmap v82: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.8 MiB/s wr, 291 op/s; 227/1467 objects degraded (15.474%); 3.1 MiB/s, 13 objects/s recovering 2024-05-20T22:27:35.330 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:35 smithi006 ceph-mon[65753]: osdmap e83: 8 total, 8 up, 8 in 2024-05-20T22:27:35.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:35 smithi078 ceph-mon[52919]: pgmap v82: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 1.8 MiB/s wr, 291 op/s; 227/1467 objects degraded (15.474%); 3.1 MiB/s, 13 objects/s recovering 2024-05-20T22:27:35.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:35 smithi078 ceph-mon[52919]: osdmap e83: 8 total, 8 up, 8 in 2024-05-20T22:27:36.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 184512 11.05 MB/sec execute 282 sec latency 1019.286 ms 2024-05-20T22:27:36.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:36 smithi006 ceph-mon[65753]: osdmap e84: 8 total, 8 up, 8 in 2024-05-20T22:27:36.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:36 smithi006 ceph-mon[65753]: pgmap v85: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 243 op/s; 227/1467 objects degraded (15.474%); 3.0 MiB/s, 5 objects/s recovering 2024-05-20T22:27:36.344 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:36 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 227/1467 objects degraded (15.474%), 10 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:36.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:36 smithi078 ceph-mon[52919]: osdmap e84: 8 total, 8 up, 8 in 2024-05-20T22:27:36.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:36 smithi078 ceph-mon[52919]: pgmap v85: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 86 active+clean; 324 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 1.6 MiB/s wr, 243 op/s; 227/1467 objects degraded (15.474%); 3.0 MiB/s, 5 objects/s recovering 2024-05-20T22:27:36.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:36 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 227/1467 objects degraded (15.474%), 10 pgs degraded (PG_DEGRADED) 2024-05-20T22:27:37.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 184781 11.04 MB/sec execute 283 sec latency 682.276 ms 2024-05-20T22:27:37.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:37 smithi006 ceph-mon[65753]: osdmap e85: 8 total, 8 up, 8 in 2024-05-20T22:27:37.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:37 smithi078 ceph-mon[52919]: osdmap e85: 8 total, 8 up, 8 in 2024-05-20T22:27:38.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 185116 11.03 MB/sec execute 284 sec latency 940.149 ms 2024-05-20T22:27:38.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:38 smithi006 ceph-mon[65753]: osdmap e86: 8 total, 8 up, 8 in 2024-05-20T22:27:38.384 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:38 smithi006 ceph-mon[65753]: pgmap v88: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 7.7 MiB/s wr, 655 op/s; 4/1707 objects degraded (0.234%); 0 B/s, 57 objects/s recovering 2024-05-20T22:27:38.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:38 smithi078 ceph-mon[52919]: osdmap e86: 8 total, 8 up, 8 in 2024-05-20T22:27:38.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:38 smithi078 ceph-mon[52919]: pgmap v88: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 7.7 MiB/s wr, 655 op/s; 4/1707 objects degraded (0.234%); 0 B/s, 57 objects/s recovering 2024-05-20T22:27:39.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 185378 11.00 MB/sec execute 285 sec latency 819.706 ms 2024-05-20T22:27:39.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:39 smithi006 ceph-mon[65753]: osdmap e87: 8 total, 8 up, 8 in 2024-05-20T22:27:39.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:39 smithi078 ceph-mon[52919]: osdmap e87: 8 total, 8 up, 8 in 2024-05-20T22:27:40.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 186008 11.04 MB/sec execute 286 sec latency 38.689 ms 2024-05-20T22:27:40.398 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:40 smithi006 ceph-mon[65753]: pgmap v90: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 7.3 MiB/s wr, 627 op/s; 4/1707 objects degraded (0.234%); 0 B/s, 55 objects/s recovering 2024-05-20T22:27:40.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:40 smithi078 ceph-mon[52919]: pgmap v90: 97 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 7.3 MiB/s wr, 627 op/s; 4/1707 objects degraded (0.234%); 0 B/s, 55 objects/s recovering 2024-05-20T22:27:41.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 186724 11.05 MB/sec execute 287 sec latency 27.491 ms 2024-05-20T22:27:41.404 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:41 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 4/1707 objects degraded (0.234%), 1 pg degraded (PG_DEGRADED) 2024-05-20T22:27:41.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:41 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 4/1707 objects degraded (0.234%), 1 pg degraded (PG_DEGRADED) 2024-05-20T22:27:42.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 187284 11.07 MB/sec execute 288 sec latency 31.564 ms 2024-05-20T22:27:42.413 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:42 smithi006 ceph-mon[65753]: pgmap v91: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 680 op/s; 0 B/s, 41 objects/s recovering 2024-05-20T22:27:42.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:42 smithi078 ceph-mon[52919]: pgmap v91: 97 pgs: 1 peering, 96 active+clean; 332 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 680 op/s; 0 B/s, 41 objects/s recovering 2024-05-20T22:27:43.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 187924 11.11 MB/sec execute 289 sec latency 38.917 ms 2024-05-20T22:27:43.359 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:43 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/1707 objects degraded (0.234%), 1 pg degraded) 2024-05-20T22:27:43.359 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:43 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:27:43.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:43 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/1707 objects degraded (0.234%), 1 pg degraded) 2024-05-20T22:27:43.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:43 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:27:44.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 188352 11.11 MB/sec execute 290 sec latency 43.583 ms 2024-05-20T22:27:44.429 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:44 smithi006 ceph-mon[65753]: pgmap v92: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.1 MiB/s wr, 811 op/s; 0 B/s, 20 objects/s recovering 2024-05-20T22:27:44.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:44 smithi078 ceph-mon[52919]: pgmap v92: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.1 MiB/s wr, 811 op/s; 0 B/s, 20 objects/s recovering 2024-05-20T22:27:45.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 188758 11.09 MB/sec execute 291 sec latency 52.831 ms 2024-05-20T22:27:46.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 189014 11.07 MB/sec execute 292 sec latency 66.476 ms 2024-05-20T22:27:46.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:46 smithi006 ceph-mon[65753]: pgmap v93: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 407 op/s; 0 B/s, 2 objects/s recovering 2024-05-20T22:27:46.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:46.742 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:46 smithi078 ceph-mon[52919]: pgmap v93: 97 pgs: 97 active+clean; 334 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 407 op/s; 0 B/s, 2 objects/s recovering 2024-05-20T22:27:46.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:46.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:27:47.077 INFO:teuthology.orchestra.run.smithi006.stdout: 5 189311 11.08 MB/sec execute 293 sec latency 53.369 ms 2024-05-20T22:27:47.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:47.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:48.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 189569 11.07 MB/sec execute 294 sec latency 63.793 ms 2024-05-20T22:27:48.532 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:48 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:48.783 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:48 smithi006 ceph-mon[65753]: pgmap v94: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 657 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:27:48.783 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:48 smithi006 ceph-mon[65753]: Upgrade: osd.1 is safe to restart 2024-05-20T22:27:48.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:48 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-05-20T22:27:48.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:48 smithi078 ceph-mon[52919]: pgmap v94: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 657 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:27:48.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:48 smithi078 ceph-mon[52919]: Upgrade: osd.1 is safe to restart 2024-05-20T22:27:49.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 189903 11.06 MB/sec execute 295 sec latency 57.237 ms 2024-05-20T22:27:50.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 190172 11.03 MB/sec execute 296 sec latency 61.049 ms 2024-05-20T22:27:50.590 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:50 smithi006 ceph-mon[65753]: pgmap v95: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 588 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:27:50.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:50 smithi078 ceph-mon[52919]: pgmap v95: 97 pgs: 97 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 588 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:27:51.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 190386 11.01 MB/sec execute 297 sec latency 81.874 ms 2024-05-20T22:27:51.722 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:51 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:51.722 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:51 smithi006 ceph-mon[65753]: Upgrade: Updating osd.1 2024-05-20T22:27:51.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:51 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:51.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:51 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:27:51.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:51 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:51.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:51 smithi006 ceph-mon[65753]: Deploying daemon osd.1 on smithi006 2024-05-20T22:27:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:51 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:51 smithi078 ceph-mon[52919]: Upgrade: Updating osd.1 2024-05-20T22:27:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:51 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:27:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:51 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-05-20T22:27:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:51 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:27:51.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:51 smithi078 ceph-mon[52919]: Deploying daemon osd.1 on smithi006 2024-05-20T22:27:52.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 190597 10.99 MB/sec execute 298 sec latency 55.220 ms 2024-05-20T22:27:53.009 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:52 smithi006 ceph-mon[65753]: pgmap v96: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 702 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:27:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 190895 10.99 MB/sec execute 299 sec latency 54.053 ms 2024-05-20T22:27:53.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:52 smithi078 ceph-mon[52919]: pgmap v96: 97 pgs: 97 active+clean; 339 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 702 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:27:54.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 191200 10.99 MB/sec execute 300 sec latency 67.125 ms 2024-05-20T22:27:55.040 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:54 smithi006 ceph-mon[65753]: pgmap v97: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 594 op/s 2024-05-20T22:27:55.078 INFO:teuthology.orchestra.run.smithi006.stdout: 5 191656 11.01 MB/sec execute 301 sec latency 34.488 ms 2024-05-20T22:27:55.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:54 smithi078 ceph-mon[52919]: pgmap v97: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 594 op/s 2024-05-20T22:27:56.079 INFO:teuthology.orchestra.run.smithi006.stdout: 5 192007 10.98 MB/sec execute 302 sec latency 35.174 ms 2024-05-20T22:27:56.305 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:56 smithi006 systemd[1]: Stopping Ceph osd.1 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:27:56.562 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:56 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[37994]: 2024-05-20T22:27:56.486+0000 7fabf45e9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:27:56.562 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:56 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[37994]: 2024-05-20T22:27:56.486+0000 7fabf45e9700 -1 osd.1 87 *** Got signal Terminated *** 2024-05-20T22:27:56.562 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:56 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[37994]: 2024-05-20T22:27:56.486+0000 7fabf45e9700 -1 osd.1 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:27:57.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:56 smithi006 ceph-mon[65753]: pgmap v98: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 438 op/s 2024-05-20T22:27:57.065 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:56 smithi006 ceph-mon[65753]: osd.1 marked itself down and dead 2024-05-20T22:27:57.079 INFO:teuthology.orchestra.run.smithi006.stdout: 5 192336 10.97 MB/sec execute 303 sec latency 136.786 ms 2024-05-20T22:27:57.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:56 smithi078 ceph-mon[52919]: pgmap v98: 97 pgs: 97 active+clean; 334 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 438 op/s 2024-05-20T22:27:57.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:56 smithi078 ceph-mon[52919]: osd.1 marked itself down and dead 2024-05-20T22:27:57.786 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:57 smithi006 podman[74014]: 2024-05-20 22:27:57.506372425 +0000 UTC m=+1.150624161 container died ac383466e6c49cb3410f27d7a2506f7bd4a68a2254a39452b6801b1d0707e513 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, version=8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, RELEASE=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container) 2024-05-20T22:27:58.084 INFO:teuthology.orchestra.run.smithi006.stdout: 5 192396 10.94 MB/sec execute 304 sec latency 1007.769 ms 2024-05-20T22:27:58.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:57 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:27:58.172 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:57 smithi006 ceph-mon[65753]: osdmap e88: 8 total, 7 up, 8 in 2024-05-20T22:27:58.172 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:58 smithi006 podman[74014]: 2024-05-20 22:27:58.041437427 +0000 UTC m=+1.685689165 container cleanup ac383466e6c49cb3410f27d7a2506f7bd4a68a2254a39452b6801b1d0707e513 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, version=8, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True) 2024-05-20T22:27:58.172 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:58 smithi006 bash[74014]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1 2024-05-20T22:27:58.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:57 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:27:58.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:57 smithi078 ceph-mon[52919]: osdmap e88: 8 total, 7 up, 8 in 2024-05-20T22:27:58.424 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:58 smithi006 podman[74029]: 2024-05-20 22:27:58.365009581 +0000 UTC m=+0.859712039 container remove ac383466e6c49cb3410f27d7a2506f7bd4a68a2254a39452b6801b1d0707e513 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, version=8, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0) 2024-05-20T22:27:58.931 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:58 smithi006 podman[74115]: 2024-05-20 22:27:58.825410942 +0000 UTC m=+0.030147079 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:27:59.084 INFO:teuthology.orchestra.run.smithi006.stdout: 5 192708 10.94 MB/sec execute 305 sec latency 37.684 ms 2024-05-20T22:27:59.150 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:58 smithi006 ceph-mon[65753]: pgmap v100: 97 pgs: 14 stale+active+clean, 83 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-05-20T22:27:59.150 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:27:58 smithi006 ceph-mon[65753]: osdmap e89: 8 total, 7 up, 8 in 2024-05-20T22:27:59.182 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:59 smithi006 podman[74115]: 2024-05-20 22:27:59.166143867 +0000 UTC m=+0.370879995 container create 60df46c6c65d876317636f2e4ebfc6592a1631bbeae105e88def0b283622c44f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-20T22:27:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:58 smithi078 ceph-mon[52919]: pgmap v100: 97 pgs: 14 stale+active+clean, 83 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 440 op/s 2024-05-20T22:27:59.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:27:58 smithi078 ceph-mon[52919]: osdmap e89: 8 total, 7 up, 8 in 2024-05-20T22:27:59.702 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:59 smithi006 podman[74115]: 2024-05-20 22:27:59.543663888 +0000 UTC m=+0.748400021 container init 60df46c6c65d876317636f2e4ebfc6592a1631bbeae105e88def0b283622c44f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True) 2024-05-20T22:27:59.702 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:59 smithi006 podman[74115]: 2024-05-20 22:27:59.550452655 +0000 UTC m=+0.755188786 container start 60df46c6c65d876317636f2e4ebfc6592a1631bbeae105e88def0b283622c44f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:27:59.968 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:59 smithi006 podman[74115]: 2024-05-20 22:27:59.742787066 +0000 UTC m=+0.947523199 container attach 60df46c6c65d876317636f2e4ebfc6592a1631bbeae105e88def0b283622c44f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:27:59.968 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:27:59 smithi006 podman[74115]: 2024-05-20 22:27:59.953742266 +0000 UTC m=+1.158478404 container died 60df46c6c65d876317636f2e4ebfc6592a1631bbeae105e88def0b283622c44f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, io.buildah.version=1.33.5) 2024-05-20T22:28:00.084 INFO:teuthology.orchestra.run.smithi006.stdout: 5 193067 10.93 MB/sec execute 306 sec latency 51.930 ms 2024-05-20T22:28:00.977 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:00 smithi006 podman[74142]: 2024-05-20 22:28:00.900366808 +0000 UTC m=+0.931403964 container remove 60df46c6c65d876317636f2e4ebfc6592a1631bbeae105e88def0b283622c44f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-20T22:28:00.977 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:00 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.1.service: Deactivated successfully. 2024-05-20T22:28:00.977 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:00 smithi006 systemd[1]: Stopped Ceph osd.1 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:28:00.977 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:00 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.1.service: Consumed 3min 28.903s CPU time. 2024-05-20T22:28:00.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:00 smithi006 ceph-mon[65753]: pgmap v102: 97 pgs: 14 stale+active+clean, 83 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 318 op/s 2024-05-20T22:28:00.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:00.978 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:01.085 INFO:teuthology.orchestra.run.smithi006.stdout: 5 193429 10.94 MB/sec execute 307 sec latency 68.285 ms 2024-05-20T22:28:01.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:00 smithi078 ceph-mon[52919]: pgmap v102: 97 pgs: 14 stale+active+clean, 83 active+clean; 338 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 318 op/s 2024-05-20T22:28:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:01.479 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:01 smithi006 systemd[1]: Starting Ceph osd.1 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:28:01.730 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:01 smithi006 podman[74237]: 2024-05-20 22:28:01.446144747 +0000 UTC m=+0.026986684 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:28:01.730 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:01 smithi006 podman[74237]: 2024-05-20 22:28:01.619352151 +0000 UTC m=+0.200194083 container create bbd097901d6c32263fd2592c0378657ff6f148a35badc2a74c7c24c0aa57b0f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514) 2024-05-20T22:28:02.084 INFO:teuthology.orchestra.run.smithi006.stdout: 5 193887 10.93 MB/sec execute 308 sec latency 49.810 ms 2024-05-20T22:28:02.232 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:02 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 165/1626 objects degraded (10.148%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:02.232 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 podman[74237]: 2024-05-20 22:28:02.039489739 +0000 UTC m=+0.620331666 container init bbd097901d6c32263fd2592c0378657ff6f148a35badc2a74c7c24c0aa57b0f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:28:02.232 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 podman[74237]: 2024-05-20 22:28:02.052124232 +0000 UTC m=+0.632966179 container start bbd097901d6c32263fd2592c0378657ff6f148a35badc2a74c7c24c0aa57b0f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-05-20T22:28:02.232 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 podman[74237]: 2024-05-20 22:28:02.219292604 +0000 UTC m=+0.800134537 container attach bbd097901d6c32263fd2592c0378657ff6f148a35badc2a74c7c24c0aa57b0f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:28:02.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:02 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 165/1626 objects degraded (10.148%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:02.742 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate[74250]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:28:02.742 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate[74250]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T22:28:02.742 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 bash[74237]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 bash[74237]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 bash[74237]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 bash[74237]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 bash[74237]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 bash[74237]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 bash[74237]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate[74250]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate[74250]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate[74250]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate[74250]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate[74250]: --> ceph-volume raw activate successful for osd ID: 1 2024-05-20T22:28:02.743 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:02 smithi006 podman[74237]: 2024-05-20 22:28:02.725055586 +0000 UTC m=+1.305897523 container died bbd097901d6c32263fd2592c0378657ff6f148a35badc2a74c7c24c0aa57b0f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-20T22:28:03.084 INFO:teuthology.orchestra.run.smithi006.stdout: 5 194205 10.91 MB/sec execute 309 sec latency 40.108 ms 2024-05-20T22:28:03.260 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:03 smithi006 ceph-mon[65753]: pgmap v103: 97 pgs: 19 active+undersized+degraded, 1 stale+active+clean, 16 active+undersized, 61 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 382 op/s; 165/1626 objects degraded (10.148%) 2024-05-20T22:28:03.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:03 smithi078 ceph-mon[52919]: pgmap v103: 97 pgs: 19 active+undersized+degraded, 1 stale+active+clean, 16 active+undersized, 61 active+clean; 332 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 382 op/s; 165/1626 objects degraded (10.148%) 2024-05-20T22:28:03.762 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:03 smithi006 podman[74386]: 2024-05-20 22:28:03.686048061 +0000 UTC m=+0.950415701 container remove bbd097901d6c32263fd2592c0378657ff6f148a35badc2a74c7c24c0aa57b0f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD) 2024-05-20T22:28:04.020 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:04 smithi006 podman[74461]: 2024-05-20 22:28:03.912191565 +0000 UTC m=+0.035730514 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:28:04.084 INFO:teuthology.orchestra.run.smithi006.stdout: 5 194741 10.94 MB/sec execute 310 sec latency 33.353 ms 2024-05-20T22:28:04.137 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:28:04.272 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:04 smithi006 podman[74461]: 2024-05-20 22:28:04.039558256 +0000 UTC m=+0.163097178 container create e235fe14466d0bc0d43c1b373792e48653c07fd4d45e584e8b40efa4860ea08a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-20T22:28:04.524 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:04 smithi006 podman[74461]: 2024-05-20 22:28:04.442022599 +0000 UTC m=+0.565561543 container init e235fe14466d0bc0d43c1b373792e48653c07fd4d45e584e8b40efa4860ea08a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T22:28:04.524 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:04 smithi006 podman[74461]: 2024-05-20 22:28:04.453931872 +0000 UTC m=+0.577470807 container start e235fe14466d0bc0d43c1b373792e48653c07fd4d45e584e8b40efa4860ea08a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-05-20T22:28:04.524 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:04 smithi006 ceph-osd[74504]: -- 172.21.15.6:0/3500843090 <== mon.1 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563dd1b23680 con 0x563dd1aecc00 2024-05-20T22:28:04.776 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:04 smithi006 bash[74461]: e235fe14466d0bc0d43c1b373792e48653c07fd4d45e584e8b40efa4860ea08a 2024-05-20T22:28:04.776 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:04 smithi006 systemd[1]: Started Ceph osd.1 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:28:05.027 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:05 smithi006 ceph-mon[65753]: pgmap v104: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 190/1623 objects degraded (11.707%) 2024-05-20T22:28:05.027 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:05 smithi006 ceph-mon[65753]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:05.027 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:05.027 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:05.027 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:05.085 INFO:teuthology.orchestra.run.smithi006.stdout: 5 195263 10.96 MB/sec execute 311 sec latency 33.150 ms 2024-05-20T22:28:05.279 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:05 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[74480]: 2024-05-20T22:28:05.029+0000 7f46f36977c0 -1 Falling back to public interface 2024-05-20T22:28:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:05 smithi078 ceph-mon[52919]: pgmap v104: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 190/1623 objects degraded (11.707%) 2024-05-20T22:28:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:05 smithi078 ceph-mon[52919]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:05.450 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:28:05.450 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (4m) 39s ago 13m 14.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (91s) 39s ago 13m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (84s) 79s ago 12m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (4m) 39s ago 13m 44.0M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (7m) 39s ago 7m 36.7M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (7m) 79s ago 7m 13.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (4m) 39s ago 14m 502M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (3m) 79s ago 12m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (3m) 39s ago 15m 46.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (2m) 79s ago 12m 35.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (7m) 39s ago 7m 290M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (7m) 79s ago 7m 87.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (4m) 39s ago 13m 16.0M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 79s ago 12m 18.4M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:28:05.451 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (50s) 39s ago 12m 12.5M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 starting - - - 1937M 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (11m) 39s ago 11m 403M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (10m) 39s ago 10m 543M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (10m) 79s ago 10m 545M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (10m) 79s ago 10m 356M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (9m) 79s ago 9m 543M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (9m) 79s ago 9m 483M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:28:05.452 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (3m) 39s ago 12m 67.5M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:28:06.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:06 smithi006 ceph-mon[65753]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:06.085 INFO:teuthology.orchestra.run.smithi006.stdout: 5 195667 10.94 MB/sec execute 312 sec latency 29.171 ms 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:28:06.099 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 5 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:28:06.100 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:28:06.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:06 smithi078 ceph-mon[52919]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:28:06.686 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:28:06.687 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:28:06.687 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "7/23 daemons upgraded", 2024-05-20T22:28:06.687 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:28:06.687 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:28:06.687 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:28:07.044 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:07 smithi006 ceph-mon[65753]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:07.044 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:07 smithi006 ceph-mon[65753]: pgmap v105: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 MiB/s wr, 257 op/s; 190/1623 objects degraded (11.707%) 2024-05-20T22:28:07.044 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:07 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3135898542' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:07.084 INFO:teuthology.orchestra.run.smithi006.stdout: 5 196019 10.93 MB/sec execute 313 sec latency 51.043 ms 2024-05-20T22:28:07.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:07 smithi078 ceph-mon[52919]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:07.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:07 smithi078 ceph-mon[52919]: pgmap v105: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 335 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 MiB/s wr, 257 op/s; 190/1623 objects degraded (11.707%) 2024-05-20T22:28:07.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:07 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3135898542' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:07.402 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 190/1623 objects degraded (11.707%), 23 pgs degraded 2024-05-20T22:28:07.402 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-05-20T22:28:07.402 INFO:teuthology.orchestra.run.smithi006.stdout: osd.1 (root=default,host=smithi006) is down 2024-05-20T22:28:07.402 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 190/1623 objects degraded (11.707%), 23 pgs degraded 2024-05-20T22:28:07.402 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-05-20T22:28:07.403 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-05-20T22:28:07.404 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-05-20T22:28:08.050 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:08 smithi006 ceph-mon[65753]: from='client.44197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:08.050 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:08 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2985030400' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:28:08.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 196419 10.94 MB/sec execute 314 sec latency 64.949 ms 2024-05-20T22:28:08.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:08 smithi078 ceph-mon[52919]: from='client.44197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:08.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:08 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2985030400' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:28:09.063 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:09 smithi006 ceph-mon[65753]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 517 op/s; 183/1692 objects degraded (10.816%) 2024-05-20T22:28:09.063 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:09 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 183/1692 objects degraded (10.816%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:09.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 196873 10.95 MB/sec execute 315 sec latency 29.174 ms 2024-05-20T22:28:09.297 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:09 smithi078 ceph-mon[52919]: pgmap v106: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 517 op/s; 183/1692 objects degraded (10.816%) 2024-05-20T22:28:09.298 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:09 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 183/1692 objects degraded (10.816%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:10.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 197190 10.93 MB/sec execute 316 sec latency 51.996 ms 2024-05-20T22:28:11.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 197528 10.93 MB/sec execute 317 sec latency 52.470 ms 2024-05-20T22:28:11.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:11 smithi006 ceph-mon[65753]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 447 op/s; 183/1692 objects degraded (10.816%) 2024-05-20T22:28:11.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:11 smithi078 ceph-mon[52919]: pgmap v107: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 338 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 447 op/s; 183/1692 objects degraded (10.816%) 2024-05-20T22:28:12.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 197920 10.93 MB/sec execute 318 sec latency 32.291 ms 2024-05-20T22:28:12.115 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:12 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[74480]: 2024-05-20T22:28:12.051+0000 7f46f36977c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-05-20T22:28:13.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 198268 10.93 MB/sec execute 319 sec latency 78.684 ms 2024-05-20T22:28:13.157 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:13 smithi006 ceph-mon[65753]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 585 op/s; 181/1641 objects degraded (11.030%) 2024-05-20T22:28:13.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:13 smithi078 ceph-mon[52919]: pgmap v108: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 333 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 585 op/s; 181/1641 objects degraded (11.030%) 2024-05-20T22:28:13.407 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[74480]: 2024-05-20T22:28:13.397+0000 7f46f36977c0 -1 osd.1 87 log_to_monitors true 2024-05-20T22:28:14.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 198691 10.94 MB/sec execute 320 sec latency 46.503 ms 2024-05-20T22:28:14.189 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:14 smithi006 ceph-mon[65753]: from='osd.1 [v2:172.21.15.6:6810/1518034162,v1:172.21.15.6:6811/1518034162]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:28:14.190 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:14 smithi006 ceph-mon[65753]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:28:14.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:14 smithi078 ceph-mon[52919]: from='osd.1 [v2:172.21.15.6:6810/1518034162,v1:172.21.15.6:6811/1518034162]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:28:14.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:14 smithi078 ceph-mon[52919]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-05-20T22:28:14.718 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:28:14 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[74480]: 2024-05-20T22:28:14.594+0000 7f46e0ae6700 -1 osd.1 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:28:15.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 198972 10.92 MB/sec execute 321 sec latency 117.552 ms 2024-05-20T22:28:15.222 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:15 smithi006 ceph-mon[65753]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 622 op/s; 180/1653 objects degraded (10.889%) 2024-05-20T22:28:15.222 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:15 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 180/1653 objects degraded (10.889%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:15.222 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:15 smithi006 ceph-mon[65753]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:28:15.222 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:15 smithi006 ceph-mon[65753]: osdmap e90: 8 total, 7 up, 8 in 2024-05-20T22:28:15.223 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:15 smithi006 ceph-mon[65753]: from='osd.1 [v2:172.21.15.6:6810/1518034162,v1:172.21.15.6:6811/1518034162]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:28:15.223 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:15 smithi006 ceph-mon[65753]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:28:15.223 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:15 smithi006 ceph-mon[65753]: from='osd.1 ' entity='osd.1' 2024-05-20T22:28:15.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:15 smithi078 ceph-mon[52919]: pgmap v109: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 336 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 622 op/s; 180/1653 objects degraded (10.889%) 2024-05-20T22:28:15.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:15 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 180/1653 objects degraded (10.889%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:15.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:15 smithi078 ceph-mon[52919]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-05-20T22:28:15.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:15 smithi078 ceph-mon[52919]: osdmap e90: 8 total, 7 up, 8 in 2024-05-20T22:28:15.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:15 smithi078 ceph-mon[52919]: from='osd.1 [v2:172.21.15.6:6810/1518034162,v1:172.21.15.6:6811/1518034162]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:28:15.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:15 smithi078 ceph-mon[52919]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:28:15.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:15 smithi078 ceph-mon[52919]: from='osd.1 ' entity='osd.1' 2024-05-20T22:28:16.087 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199221 10.90 MB/sec execute 322 sec latency 412.316 ms 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: osd.1 [v2:172.21.15.6:6810/1518034162,v1:172.21.15.6:6811/1518034162] boot 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: osdmap e91: 8 total, 8 up, 8 in 2024-05-20T22:28:16.254 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:16.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:28:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: osd.1 [v2:172.21.15.6:6810/1518034162,v1:172.21.15.6:6811/1518034162] boot 2024-05-20T22:28:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: osdmap e91: 8 total, 8 up, 8 in 2024-05-20T22:28:16.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-05-20T22:28:17.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199287 10.87 MB/sec execute 323 sec latency 1375.733 ms 2024-05-20T22:28:17.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:17 smithi006 ceph-mon[65753]: pgmap v111: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.1 MiB/s wr, 593 op/s; 180/1653 objects degraded (10.889%) 2024-05-20T22:28:17.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:17.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:17.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:17 smithi006 ceph-mon[65753]: osdmap e92: 8 total, 8 up, 8 in 2024-05-20T22:28:17.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:17 smithi078 ceph-mon[52919]: pgmap v111: 97 pgs: 23 active+undersized+degraded, 16 active+undersized, 58 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.1 MiB/s wr, 593 op/s; 180/1653 objects degraded (10.889%) 2024-05-20T22:28:17.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:17.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:17.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:17 smithi078 ceph-mon[52919]: osdmap e92: 8 total, 8 up, 8 in 2024-05-20T22:28:18.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199361 10.84 MB/sec execute 324 sec latency 1932.700 ms 2024-05-20T22:28:18.588 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:18 smithi006 ceph-mon[65753]: pgmap v114: 97 pgs: 25 peering, 9 active+undersized+degraded, 5 active+undersized, 58 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 564 op/s; 72/1662 objects degraded (4.332%) 2024-05-20T22:28:18.589 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:18 smithi006 ceph-mon[65753]: osdmap e93: 8 total, 8 up, 8 in 2024-05-20T22:28:18.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:18 smithi078 ceph-mon[52919]: pgmap v114: 97 pgs: 25 peering, 9 active+undersized+degraded, 5 active+undersized, 58 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 564 op/s; 72/1662 objects degraded (4.332%) 2024-05-20T22:28:18.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:18 smithi078 ceph-mon[52919]: osdmap e93: 8 total, 8 up, 8 in 2024-05-20T22:28:19.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199663 10.83 MB/sec execute 325 sec latency 73.454 ms 2024-05-20T22:28:19.618 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:19 smithi006 ceph-mon[65753]: osdmap e94: 8 total, 8 up, 8 in 2024-05-20T22:28:19.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:19 smithi078 ceph-mon[52919]: osdmap e94: 8 total, 8 up, 8 in 2024-05-20T22:28:20.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 199890 10.82 MB/sec execute 326 sec latency 595.728 ms 2024-05-20T22:28:20.632 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:20 smithi006 ceph-mon[65753]: pgmap v117: 97 pgs: 26 peering, 3 active+undersized+degraded, 68 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 568 op/s; 45/1662 objects degraded (2.708%); 22 B/s, 0 objects/s recovering 2024-05-20T22:28:20.632 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:20 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 45/1662 objects degraded (2.708%), 3 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:20.632 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:20 smithi006 ceph-mon[65753]: osdmap e95: 8 total, 8 up, 8 in 2024-05-20T22:28:20.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:20 smithi078 ceph-mon[52919]: pgmap v117: 97 pgs: 26 peering, 3 active+undersized+degraded, 68 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 568 op/s; 45/1662 objects degraded (2.708%); 22 B/s, 0 objects/s recovering 2024-05-20T22:28:20.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:20 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 45/1662 objects degraded (2.708%), 3 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:20.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:20 smithi078 ceph-mon[52919]: osdmap e95: 8 total, 8 up, 8 in 2024-05-20T22:28:21.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 200473 10.84 MB/sec execute 327 sec latency 966.568 ms 2024-05-20T22:28:21.640 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:21 smithi006 ceph-mon[65753]: osdmap e96: 8 total, 8 up, 8 in 2024-05-20T22:28:21.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:21 smithi078 ceph-mon[52919]: osdmap e96: 8 total, 8 up, 8 in 2024-05-20T22:28:22.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 200963 10.83 MB/sec execute 328 sec latency 891.309 ms 2024-05-20T22:28:22.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:22 smithi006 ceph-mon[65753]: pgmap v120: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 535 op/s; 110/1623 objects degraded (6.778%); 1.1 MiB/s, 32 objects/s recovering 2024-05-20T22:28:22.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:22 smithi006 ceph-mon[65753]: osdmap e97: 8 total, 8 up, 8 in 2024-05-20T22:28:22.651 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:22 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:22.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:22 smithi078 ceph-mon[52919]: pgmap v120: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 535 op/s; 110/1623 objects degraded (6.778%); 1.1 MiB/s, 32 objects/s recovering 2024-05-20T22:28:22.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:22 smithi078 ceph-mon[52919]: osdmap e97: 8 total, 8 up, 8 in 2024-05-20T22:28:22.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:22 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:23.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 201446 10.86 MB/sec execute 329 sec latency 325.268 ms 2024-05-20T22:28:23.685 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:23.685 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: osdmap e98: 8 total, 8 up, 8 in 2024-05-20T22:28:23.685 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:23.685 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:23.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:23 smithi006 ceph-mon[65753]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: osdmap e98: 8 total, 8 up, 8 in 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:23.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:23.898 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:23 smithi078 ceph-mon[52919]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-20T22:28:24.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 201821 10.86 MB/sec execute 330 sec latency 871.303 ms 2024-05-20T22:28:24.715 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:24 smithi006 ceph-mon[65753]: pgmap v123: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 532 op/s; 110/1623 objects degraded (6.778%); 1.1 MiB/s, 32 objects/s recovering 2024-05-20T22:28:24.715 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:24 smithi006 ceph-mon[65753]: osdmap e99: 8 total, 8 up, 8 in 2024-05-20T22:28:24.715 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:24 smithi006 ceph-mon[65753]: osdmap e100: 8 total, 8 up, 8 in 2024-05-20T22:28:24.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:24 smithi078 ceph-mon[52919]: pgmap v123: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+undersized+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 88 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.0 MiB/s wr, 532 op/s; 110/1623 objects degraded (6.778%); 1.1 MiB/s, 32 objects/s recovering 2024-05-20T22:28:24.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:24 smithi078 ceph-mon[52919]: osdmap e99: 8 total, 8 up, 8 in 2024-05-20T22:28:24.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:24 smithi078 ceph-mon[52919]: osdmap e100: 8 total, 8 up, 8 in 2024-05-20T22:28:25.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 202126 10.85 MB/sec execute 331 sec latency 917.900 ms 2024-05-20T22:28:25.780 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:25 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 110/1623 objects degraded (6.778%), 7 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:25.780 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:25 smithi006 ceph-mon[65753]: osdmap e101: 8 total, 8 up, 8 in 2024-05-20T22:28:25.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:25 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 110/1623 objects degraded (6.778%), 7 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:25.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:25 smithi078 ceph-mon[52919]: osdmap e101: 8 total, 8 up, 8 in 2024-05-20T22:28:26.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 202465 10.86 MB/sec execute 332 sec latency 954.942 ms 2024-05-20T22:28:26.792 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:26 smithi006 ceph-mon[65753]: pgmap v126: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 90 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 759 KiB/s wr, 30 op/s; 72/1632 objects degraded (4.412%); 1024 KiB/s, 71 keys/s, 13 objects/s recovering 2024-05-20T22:28:26.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:26 smithi078 ceph-mon[52919]: pgmap v126: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 90 active+clean; 338 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 759 KiB/s wr, 30 op/s; 72/1632 objects degraded (4.412%); 1024 KiB/s, 71 keys/s, 13 objects/s recovering 2024-05-20T22:28:27.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 202811 10.84 MB/sec execute 333 sec latency 37.645 ms 2024-05-20T22:28:28.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 203213 10.83 MB/sec execute 334 sec latency 73.578 ms 2024-05-20T22:28:28.863 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:28 smithi006 ceph-mon[65753]: pgmap v128: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 95 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.4 MiB/s wr, 576 op/s; 24/1722 objects degraded (1.394%); 838 KiB/s, 58 keys/s, 21 objects/s recovering 2024-05-20T22:28:28.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:28 smithi078 ceph-mon[52919]: pgmap v128: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 95 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.4 MiB/s wr, 576 op/s; 24/1722 objects degraded (1.394%); 838 KiB/s, 58 keys/s, 21 objects/s recovering 2024-05-20T22:28:29.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 203588 10.83 MB/sec execute 335 sec latency 39.950 ms 2024-05-20T22:28:29.888 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:29 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1722 objects degraded (1.394%), 1 pg degraded) 2024-05-20T22:28:29.889 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:29 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:28:30.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 204077 10.84 MB/sec execute 336 sec latency 50.266 ms 2024-05-20T22:28:30.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:29 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1722 objects degraded (1.394%), 1 pg degraded) 2024-05-20T22:28:30.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:29 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:28:30.903 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:30 smithi006 ceph-mon[65753]: pgmap v129: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.6 MiB/s wr, 485 op/s; 682 KiB/s, 47 keys/s, 21 objects/s recovering 2024-05-20T22:28:30.903 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:30.903 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:31.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 204579 10.84 MB/sec execute 337 sec latency 40.201 ms 2024-05-20T22:28:31.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:30 smithi078 ceph-mon[52919]: pgmap v129: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.6 MiB/s wr, 485 op/s; 682 KiB/s, 47 keys/s, 21 objects/s recovering 2024-05-20T22:28:31.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:31.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:32.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 204952 10.84 MB/sec execute 338 sec latency 34.014 ms 2024-05-20T22:28:32.930 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:32 smithi006 ceph-mon[65753]: pgmap v130: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 749 op/s; 519 KiB/s, 36 keys/s, 16 objects/s recovering 2024-05-20T22:28:33.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 205430 10.85 MB/sec execute 339 sec latency 33.278 ms 2024-05-20T22:28:33.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:32 smithi078 ceph-mon[52919]: pgmap v130: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 749 op/s; 519 KiB/s, 36 keys/s, 16 objects/s recovering 2024-05-20T22:28:34.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 205862 10.86 MB/sec execute 340 sec latency 45.501 ms 2024-05-20T22:28:34.703 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:34 smithi006 ceph-mon[65753]: pgmap v131: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 651 op/s; 0 B/s, 8 objects/s recovering 2024-05-20T22:28:35.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206270 10.87 MB/sec execute 341 sec latency 28.947 ms 2024-05-20T22:28:35.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:34 smithi078 ceph-mon[52919]: pgmap v131: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 651 op/s; 0 B/s, 8 objects/s recovering 2024-05-20T22:28:36.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206701 10.86 MB/sec execute 342 sec latency 52.967 ms 2024-05-20T22:28:36.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:36 smithi006 ceph-mon[65753]: pgmap v132: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 590 op/s; 0 B/s, 7 objects/s recovering 2024-05-20T22:28:36.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:36 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:36.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:36 smithi078 ceph-mon[52919]: pgmap v132: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 590 op/s; 0 B/s, 7 objects/s recovering 2024-05-20T22:28:36.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:36 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:37.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 206940 10.84 MB/sec execute 343 sec latency 52.930 ms 2024-05-20T22:28:37.852 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:28:38.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 207121 10.82 MB/sec execute 344 sec latency 78.951 ms 2024-05-20T22:28:38.780 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:38 smithi006 ceph-mon[65753]: pgmap v133: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 743 op/s; 0 B/s, 6 objects/s recovering 2024-05-20T22:28:38.780 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:38.780 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:38 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:38.780 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:38 smithi006 ceph-mon[65753]: Upgrade: osd.2 is safe to restart 2024-05-20T22:28:38.780 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:38 smithi006 ceph-mon[65753]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:38.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:38 smithi078 ceph-mon[52919]: pgmap v133: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 743 op/s; 0 B/s, 6 objects/s recovering 2024-05-20T22:28:38.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:38.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:38 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-05-20T22:28:38.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:38 smithi078 ceph-mon[52919]: Upgrade: osd.2 is safe to restart 2024-05-20T22:28:38.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:38 smithi078 ceph-mon[52919]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:39.065 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (5m) 23s ago 14m 14.2M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (2m) 23s ago 14m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (118s) 112s ago 13m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (5m) 23s ago 13m 45.5M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (8m) 23s ago 8m 36.9M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (8m) 112s ago 8m 13.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (4m) 23s ago 15m 506M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (3m) 112s ago 13m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (3m) 23s ago 15m 49.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (3m) 112s ago 13m 35.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (7m) 23s ago 7m 286M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:28:39.066 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (7m) 112s ago 7m 87.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (5m) 23s ago 13m 14.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (4m) 112s ago 13m 18.4M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (84s) 23s ago 12m 241M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (35s) 23s ago 12m 11.7M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (11m) 23s ago 11m 480M 1937M 17.2.0 e1d6a67b021e ddd93e734590 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (11m) 23s ago 11m 603M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (10m) 112s ago 10m 545M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (10m) 112s ago 10m 356M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (10m) 112s ago 10m 543M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (9m) 112s ago 9m 483M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:28:39.067 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (4m) 23s ago 12m 67.6M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:28:39.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 207456 10.82 MB/sec execute 345 sec latency 63.772 ms 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:28:39.639 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 6 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:28:39.640 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:28:39.851 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:39 smithi006 ceph-mon[65753]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:39.851 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:39 smithi006 ceph-mon[65753]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:39.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:39 smithi078 ceph-mon[52919]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:39.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:39 smithi078 ceph-mon[52919]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:40.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 207660 10.80 MB/sec execute 346 sec latency 75.646 ms 2024-05-20T22:28:40.169 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:28:40.170 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:28:40.170 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:28:40.170 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:28:40.170 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:28:40.170 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:28:40.171 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:28:40.171 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:28:40.171 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:28:40.171 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "8/23 daemons upgraded", 2024-05-20T22:28:40.171 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:28:40.171 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:28:40.171 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:28:40.615 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: pgmap v134: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 519 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:28:40.615 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2219613127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:40.615 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: from='client.44221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:40.615 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: Upgrade: Updating osd.2 2024-05-20T22:28:40.615 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:40.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:28:40.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:40.616 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:40 smithi006 ceph-mon[65753]: Deploying daemon osd.2 on smithi006 2024-05-20T22:28:40.697 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:28:40.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: pgmap v134: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 519 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:28:40.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2219613127' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:28:40.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: from='client.44221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:28:40.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: Upgrade: Updating osd.2 2024-05-20T22:28:40.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:40.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-05-20T22:28:40.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:28:40.897 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:40 smithi078 ceph-mon[52919]: Deploying daemon osd.2 on smithi006 2024-05-20T22:28:41.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 207946 10.78 MB/sec execute 347 sec latency 106.158 ms 2024-05-20T22:28:41.634 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:41 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1770985910' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:28:41.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:41 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1770985910' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:28:42.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 208274 10.78 MB/sec execute 348 sec latency 47.422 ms 2024-05-20T22:28:42.661 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:42 smithi006 ceph-mon[65753]: pgmap v135: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-05-20T22:28:42.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:42 smithi078 ceph-mon[52919]: pgmap v135: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.3 MiB/s wr, 653 op/s 2024-05-20T22:28:43.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 208585 10.78 MB/sec execute 349 sec latency 59.719 ms 2024-05-20T22:28:44.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 208952 10.78 MB/sec execute 350 sec latency 61.445 ms 2024-05-20T22:28:44.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:44 smithi006 ceph-mon[65753]: pgmap v136: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-05-20T22:28:44.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:44 smithi078 ceph-mon[52919]: pgmap v136: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.2 MiB/s wr, 404 op/s 2024-05-20T22:28:45.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 209305 10.79 MB/sec execute 351 sec latency 46.398 ms 2024-05-20T22:28:45.463 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:45 smithi006 systemd[1]: Stopping Ceph osd.2 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:28:45.714 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:45 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[41821]: 2024-05-20T22:28:45.701+0000 7f041a377700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:28:45.715 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:45 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[41821]: 2024-05-20T22:28:45.701+0000 7f041a377700 -1 osd.2 101 *** Got signal Terminated *** 2024-05-20T22:28:45.715 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:45 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[41821]: 2024-05-20T22:28:45.701+0000 7f041a377700 -1 osd.2 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:28:45.715 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:45.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:28:46.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 209642 10.78 MB/sec execute 352 sec latency 53.456 ms 2024-05-20T22:28:46.726 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:46 smithi006 ceph-mon[65753]: pgmap v137: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 489 op/s 2024-05-20T22:28:46.726 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:46 smithi006 ceph-mon[65753]: osd.2 marked itself down and dead 2024-05-20T22:28:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:46 smithi078 ceph-mon[52919]: pgmap v137: 97 pgs: 97 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 489 op/s 2024-05-20T22:28:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:46 smithi078 ceph-mon[52919]: osd.2 marked itself down and dead 2024-05-20T22:28:47.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 210038 10.78 MB/sec execute 353 sec latency 454.379 ms 2024-05-20T22:28:47.536 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:47 smithi006 podman[76713]: 2024-05-20 22:28:47.288898655 +0000 UTC m=+1.729853365 container died ddd93e7345903cfa723a6da34570b9e7bf3cf2acd5a32b809fc95427641dc5bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, release=754, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream) 2024-05-20T22:28:47.859 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:47 smithi006 podman[76713]: 2024-05-20 22:28:47.837957223 +0000 UTC m=+2.278911927 container cleanup ddd93e7345903cfa723a6da34570b9e7bf3cf2acd5a32b809fc95427641dc5bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_BRANCH=HEAD, name=centos-stream, architecture=x86_64, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:28:47.860 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:47 smithi006 bash[76713]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2 2024-05-20T22:28:47.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:47 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:28:47.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:47 smithi006 ceph-mon[65753]: osdmap e102: 8 total, 7 up, 8 in 2024-05-20T22:28:47.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:47 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:28:47.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:47 smithi078 ceph-mon[52919]: osdmap e102: 8 total, 7 up, 8 in 2024-05-20T22:28:48.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 210199 10.76 MB/sec execute 354 sec latency 1030.039 ms 2024-05-20T22:28:48.373 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:48 smithi006 podman[76730]: 2024-05-20 22:28:48.208339864 +0000 UTC m=+0.916200841 container remove ddd93e7345903cfa723a6da34570b9e7bf3cf2acd5a32b809fc95427641dc5bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-05-20T22:28:48.631 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:48 smithi006 ceph-mon[65753]: pgmap v139: 97 pgs: 5 stale+active+clean, 92 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 430 op/s 2024-05-20T22:28:48.631 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:48 smithi006 ceph-mon[65753]: osdmap e103: 8 total, 7 up, 8 in 2024-05-20T22:28:48.888 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:48 smithi006 podman[76810]: 2024-05-20 22:28:48.742475903 +0000 UTC m=+0.028131521 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:28:48.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:48 smithi078 ceph-mon[52919]: pgmap v139: 97 pgs: 5 stale+active+clean, 92 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 430 op/s 2024-05-20T22:28:48.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:48 smithi078 ceph-mon[52919]: osdmap e103: 8 total, 7 up, 8 in 2024-05-20T22:28:48.910 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:48 smithi006 podman[76810]: 2024-05-20 22:28:48.892514915 +0000 UTC m=+0.178170516 container create e7ad2cbc0211e1dc53eff496209d6cdded8b14e45c2133b25059f1a7c0d8778f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-05-20T22:28:49.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 210555 10.75 MB/sec execute 355 sec latency 38.523 ms 2024-05-20T22:28:49.404 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:49 smithi006 podman[76810]: 2024-05-20 22:28:49.257778199 +0000 UTC m=+0.543433797 container init e7ad2cbc0211e1dc53eff496209d6cdded8b14e45c2133b25059f1a7c0d8778f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-05-20T22:28:49.405 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:49 smithi006 podman[76810]: 2024-05-20 22:28:49.268903771 +0000 UTC m=+0.554559381 container start e7ad2cbc0211e1dc53eff496209d6cdded8b14e45c2133b25059f1a7c0d8778f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514) 2024-05-20T22:28:49.656 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:49 smithi006 podman[76810]: 2024-05-20 22:28:49.413953509 +0000 UTC m=+0.699609119 container attach e7ad2cbc0211e1dc53eff496209d6cdded8b14e45c2133b25059f1a7c0d8778f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-20T22:28:49.907 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:49 smithi006 podman[76810]: 2024-05-20 22:28:49.767083497 +0000 UTC m=+1.052739105 container died e7ad2cbc0211e1dc53eff496209d6cdded8b14e45c2133b25059f1a7c0d8778f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-20T22:28:49.908 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:49 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 33/1659 objects degraded (1.989%), 3 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:50.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 210951 10.74 MB/sec execute 356 sec latency 47.334 ms 2024-05-20T22:28:50.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:49 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 33/1659 objects degraded (1.989%), 3 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:50.924 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:50 smithi006 ceph-mon[65753]: pgmap v141: 97 pgs: 3 active+undersized+degraded, 3 stale+active+clean, 3 active+undersized, 88 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.9 MiB/s wr, 342 op/s; 33/1659 objects degraded (1.989%) 2024-05-20T22:28:50.925 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:50 smithi006 podman[76866]: 2024-05-20 22:28:50.877911663 +0000 UTC m=+1.102860465 container remove e7ad2cbc0211e1dc53eff496209d6cdded8b14e45c2133b25059f1a7c0d8778f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-05-20T22:28:50.925 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:50 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.2.service: Deactivated successfully. 2024-05-20T22:28:50.925 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:50 smithi006 systemd[1]: Stopped Ceph osd.2 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:28:50.925 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:50 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.2.service: Consumed 2min 42.504s CPU time. 2024-05-20T22:28:51.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 211406 10.74 MB/sec execute 357 sec latency 42.693 ms 2024-05-20T22:28:51.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:50 smithi078 ceph-mon[52919]: pgmap v141: 97 pgs: 3 active+undersized+degraded, 3 stale+active+clean, 3 active+undersized, 88 active+clean; 342 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.9 MiB/s wr, 342 op/s; 33/1659 objects degraded (1.989%) 2024-05-20T22:28:51.433 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:51 smithi006 systemd[1]: Starting Ceph osd.2 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:28:51.683 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:51 smithi006 podman[76961]: 2024-05-20 22:28:51.536663068 +0000 UTC m=+0.026314065 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:28:51.939 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:51 smithi006 podman[76961]: 2024-05-20 22:28:51.82248317 +0000 UTC m=+0.312134144 container create 342219c80a5482d892014e727185d7937f834b5c7148bde23de51fce023a0e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-20T22:28:52.088 INFO:teuthology.orchestra.run.smithi006.stdout: 5 211824 10.75 MB/sec execute 358 sec latency 36.695 ms 2024-05-20T22:28:52.453 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 podman[76961]: 2024-05-20 22:28:52.197178093 +0000 UTC m=+0.686829081 container init 342219c80a5482d892014e727185d7937f834b5c7148bde23de51fce023a0e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux ) 2024-05-20T22:28:52.543 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 podman[76961]: 2024-05-20 22:28:52.208223302 +0000 UTC m=+0.697874291 container start 342219c80a5482d892014e727185d7937f834b5c7148bde23de51fce023a0e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:28:52.544 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 podman[76961]: 2024-05-20 22:28:52.400077814 +0000 UTC m=+0.889728812 container attach 342219c80a5482d892014e727185d7937f834b5c7148bde23de51fce023a0e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-05-20T22:28:52.708 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:52 smithi006 ceph-mon[65753]: pgmap v142: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 660 op/s; 148/1650 objects degraded (8.970%) 2024-05-20T22:28:52.960 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate[76976]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T22:28:52.960 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate[76976]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T22:28:52.960 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 bash[76961]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T22:28:52.960 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 bash[76961]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 bash[76961]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 bash[76961]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 bash[76961]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 bash[76961]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 bash[76961]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate[76976]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate[76976]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate[76976]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate[76976]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate[76976]: --> ceph-volume raw activate successful for osd ID: 2 2024-05-20T22:28:52.961 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:52 smithi006 podman[76961]: 2024-05-20 22:28:52.890689433 +0000 UTC m=+1.380340423 container died 342219c80a5482d892014e727185d7937f834b5c7148bde23de51fce023a0e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-05-20T22:28:53.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 212275 10.76 MB/sec execute 359 sec latency 42.765 ms 2024-05-20T22:28:53.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:52 smithi078 ceph-mon[52919]: pgmap v142: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.5 MiB/s wr, 660 op/s; 148/1650 objects degraded (8.970%) 2024-05-20T22:28:54.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 212666 10.77 MB/sec execute 360 sec latency 43.590 ms 2024-05-20T22:28:54.477 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:54 smithi006 podman[77107]: 2024-05-20 22:28:54.273960381 +0000 UTC m=+1.374925776 container remove 342219c80a5482d892014e727185d7937f834b5c7148bde23de51fce023a0e8f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-05-20T22:28:54.718 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:54 smithi006 podman[77164]: 2024-05-20 22:28:54.443435622 +0000 UTC m=+0.027783921 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:28:54.718 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:54 smithi006 podman[77164]: 2024-05-20 22:28:54.634365001 +0000 UTC m=+0.218713288 container create 2b05adf2b057f702fe7ea9ece3444e4b0619cdc9dfc85e7858007292c698950d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS) 2024-05-20T22:28:54.719 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:54 smithi006 ceph-mon[65753]: pgmap v143: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.8 MiB/s wr, 533 op/s; 148/1650 objects degraded (8.970%) 2024-05-20T22:28:55.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 213082 10.78 MB/sec execute 361 sec latency 27.260 ms 2024-05-20T22:28:55.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:54 smithi078 ceph-mon[52919]: pgmap v143: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.8 MiB/s wr, 533 op/s; 148/1650 objects degraded (8.970%) 2024-05-20T22:28:55.219 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:55 smithi006 podman[77164]: 2024-05-20 22:28:55.055091325 +0000 UTC m=+0.639439619 container init 2b05adf2b057f702fe7ea9ece3444e4b0619cdc9dfc85e7858007292c698950d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, org.label-schema.build-date=20240514, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:28:55.219 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:55 smithi006 podman[77164]: 2024-05-20 22:28:55.063165526 +0000 UTC m=+0.647513826 container start 2b05adf2b057f702fe7ea9ece3444e4b0619cdc9dfc85e7858007292c698950d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:28:55.219 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:55 smithi006 ceph-osd[77183]: -- 172.21.15.6:0/2463880564 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f355e45680 con 0x55f355e0ec00 2024-05-20T22:28:55.219 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:55 smithi006 bash[77164]: 2b05adf2b057f702fe7ea9ece3444e4b0619cdc9dfc85e7858007292c698950d 2024-05-20T22:28:55.219 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:55 smithi006 systemd[1]: Started Ceph osd.2 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:28:55.728 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:28:55 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[77179]: 2024-05-20T22:28:55.632+0000 7f908ff987c0 -1 Falling back to public interface 2024-05-20T22:28:56.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 213490 10.77 MB/sec execute 362 sec latency 48.539 ms 2024-05-20T22:28:56.483 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:56 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:56.483 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:56 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:56.483 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:56 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:56.483 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:56 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 148/1650 objects degraded (8.970%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:56.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:56 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:56.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:56 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:28:56.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:56 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:28:56.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:56 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 148/1650 objects degraded (8.970%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:28:57.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 213853 10.76 MB/sec execute 363 sec latency 69.258 ms 2024-05-20T22:28:57.494 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:57 smithi006 ceph-mon[65753]: pgmap v144: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 324 op/s; 155/1677 objects degraded (9.243%) 2024-05-20T22:28:57.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:57 smithi078 ceph-mon[52919]: pgmap v144: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 324 op/s; 155/1677 objects degraded (9.243%) 2024-05-20T22:28:58.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 214282 10.76 MB/sec execute 364 sec latency 42.949 ms 2024-05-20T22:28:59.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 214711 10.75 MB/sec execute 365 sec latency 99.493 ms 2024-05-20T22:28:59.514 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:28:59 smithi006 ceph-mon[65753]: pgmap v145: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.5 MiB/s wr, 555 op/s; 158/1695 objects degraded (9.322%) 2024-05-20T22:28:59.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:28:59 smithi078 ceph-mon[52919]: pgmap v145: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.5 MiB/s wr, 555 op/s; 158/1695 objects degraded (9.322%) 2024-05-20T22:29:00.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 215094 10.74 MB/sec execute 366 sec latency 36.144 ms 2024-05-20T22:29:01.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 215430 10.75 MB/sec execute 367 sec latency 40.783 ms 2024-05-20T22:29:01.296 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:01 smithi006 ceph-mon[65753]: pgmap v146: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.2 MiB/s wr, 564 op/s; 156/1689 objects degraded (9.236%) 2024-05-20T22:29:01.297 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:01.297 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:01.297 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:01.297 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:01 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 156/1689 objects degraded (9.236%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:01.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:01 smithi078 ceph-mon[52919]: pgmap v146: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.2 MiB/s wr, 564 op/s; 156/1689 objects degraded (9.236%) 2024-05-20T22:29:01.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:01.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:01.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:01.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:01 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 156/1689 objects degraded (9.236%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:02.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 215839 10.76 MB/sec execute 368 sec latency 48.788 ms 2024-05-20T22:29:02.807 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:29:02 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[77179]: 2024-05-20T22:29:02.698+0000 7f908ff987c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-05-20T22:29:03.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 216263 10.77 MB/sec execute 369 sec latency 34.745 ms 2024-05-20T22:29:03.320 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:03 smithi006 ceph-mon[65753]: pgmap v147: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 705 op/s; 145/1608 objects degraded (9.017%) 2024-05-20T22:29:03.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:03 smithi078 ceph-mon[52919]: pgmap v147: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 705 op/s; 145/1608 objects degraded (9.017%) 2024-05-20T22:29:03.824 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:29:03 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[77179]: 2024-05-20T22:29:03.735+0000 7f908ff987c0 -1 osd.2 101 log_to_monitors true 2024-05-20T22:29:04.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 216636 10.77 MB/sec execute 370 sec latency 29.957 ms 2024-05-20T22:29:04.326 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:04 smithi006 ceph-mon[65753]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T22:29:04.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:04 smithi078 ceph-mon[52919]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-05-20T22:29:05.083 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:29:04 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[77179]: 2024-05-20T22:29:04.837+0000 7f907d3e7700 -1 osd.2 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:29:05.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217097 10.77 MB/sec execute 371 sec latency 67.041 ms 2024-05-20T22:29:05.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:05 smithi006 ceph-mon[65753]: pgmap v148: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 145/1608 objects degraded (9.017%) 2024-05-20T22:29:05.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:05 smithi006 ceph-mon[65753]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T22:29:05.335 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:05 smithi006 ceph-mon[65753]: osdmap e104: 8 total, 7 up, 8 in 2024-05-20T22:29:05.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:05 smithi006 ceph-mon[65753]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:29:05.336 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:05 smithi006 ceph-mon[65753]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' 2024-05-20T22:29:05.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:05 smithi078 ceph-mon[52919]: pgmap v148: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 343 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 145/1608 objects degraded (9.017%) 2024-05-20T22:29:05.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:05 smithi078 ceph-mon[52919]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-05-20T22:29:05.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:05 smithi078 ceph-mon[52919]: osdmap e104: 8 total, 7 up, 8 in 2024-05-20T22:29:05.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:05 smithi078 ceph-mon[52919]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:29:05.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:05 smithi078 ceph-mon[52919]: from='osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870]' entity='osd.2' 2024-05-20T22:29:06.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217534 10.76 MB/sec execute 372 sec latency 278.430 ms 2024-05-20T22:29:06.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:06 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 145/1608 objects degraded (9.017%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:06.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:06 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:29:06.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:06 smithi078 ceph-mon[52919]: osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870] boot 2024-05-20T22:29:06.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:06 smithi078 ceph-mon[52919]: osdmap e105: 8 total, 8 up, 8 in 2024-05-20T22:29:06.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:06 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:06.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:06 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:06.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:06 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:06.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:06 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 145/1608 objects degraded (9.017%), 19 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:06.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:06 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:29:06.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:06 smithi006 ceph-mon[65753]: osd.2 [v2:172.21.15.6:6818/4032261870,v1:172.21.15.6:6819/4032261870] boot 2024-05-20T22:29:06.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:06 smithi006 ceph-mon[65753]: osdmap e105: 8 total, 8 up, 8 in 2024-05-20T22:29:06.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:06 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-05-20T22:29:06.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:06 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:06.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:06 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:07.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217534 10.73 MB/sec execute 373 sec latency 1278.462 ms 2024-05-20T22:29:07.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:07 smithi078 ceph-mon[52919]: pgmap v150: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 590 op/s; 146/1614 objects degraded (9.046%) 2024-05-20T22:29:07.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:07 smithi078 ceph-mon[52919]: osdmap e106: 8 total, 8 up, 8 in 2024-05-20T22:29:07.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:07 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:07.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:07 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:07.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:07 smithi006 ceph-mon[65753]: pgmap v150: 97 pgs: 19 active+undersized+degraded, 7 active+undersized, 71 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 590 op/s; 146/1614 objects degraded (9.046%) 2024-05-20T22:29:07.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:07 smithi006 ceph-mon[65753]: osdmap e106: 8 total, 8 up, 8 in 2024-05-20T22:29:07.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:07 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:07.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:07 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:08.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 217588 10.71 MB/sec execute 374 sec latency 2112.619 ms 2024-05-20T22:29:09.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:08 smithi006 ceph-mon[65753]: pgmap v153: 97 pgs: 15 peering, 7 active+undersized+degraded, 4 active+undersized, 71 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 MiB/s wr, 474 op/s; 67/1833 objects degraded (3.655%) 2024-05-20T22:29:09.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:08 smithi006 ceph-mon[65753]: osdmap e107: 8 total, 8 up, 8 in 2024-05-20T22:29:09.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218080 10.71 MB/sec execute 375 sec latency 182.781 ms 2024-05-20T22:29:09.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:08 smithi078 ceph-mon[52919]: pgmap v153: 97 pgs: 15 peering, 7 active+undersized+degraded, 4 active+undersized, 71 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.3 MiB/s wr, 474 op/s; 67/1833 objects degraded (3.655%) 2024-05-20T22:29:09.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:08 smithi078 ceph-mon[52919]: osdmap e107: 8 total, 8 up, 8 in 2024-05-20T22:29:09.833 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:09 smithi006 ceph-mon[65753]: osdmap e108: 8 total, 8 up, 8 in 2024-05-20T22:29:10.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218370 10.70 MB/sec execute 376 sec latency 946.583 ms 2024-05-20T22:29:10.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:09 smithi078 ceph-mon[52919]: osdmap e108: 8 total, 8 up, 8 in 2024-05-20T22:29:10.843 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:10 smithi006 ceph-mon[65753]: pgmap v156: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 16 peering, 1 active+undersized+degraded, 1 active+undersized, 78 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 705 op/s; 35/1815 objects degraded (1.928%); 72 KiB/s, 6 objects/s recovering 2024-05-20T22:29:10.843 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:10 smithi006 ceph-mon[65753]: osdmap e109: 8 total, 8 up, 8 in 2024-05-20T22:29:10.844 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:10 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 35/1815 objects degraded (1.928%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:11.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218646 10.68 MB/sec execute 377 sec latency 708.937 ms 2024-05-20T22:29:11.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:10 smithi078 ceph-mon[52919]: pgmap v156: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 16 peering, 1 active+undersized+degraded, 1 active+undersized, 78 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.7 MiB/s wr, 705 op/s; 35/1815 objects degraded (1.928%); 72 KiB/s, 6 objects/s recovering 2024-05-20T22:29:11.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:10 smithi078 ceph-mon[52919]: osdmap e109: 8 total, 8 up, 8 in 2024-05-20T22:29:11.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:10 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 35/1815 objects degraded (1.928%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:11.238 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:29:11.864 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:11 smithi006 ceph-mon[65753]: osdmap e110: 8 total, 8 up, 8 in 2024-05-20T22:29:11.864 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:11 smithi006 ceph-mon[65753]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:12.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 218996 10.69 MB/sec execute 378 sec latency 831.792 ms 2024-05-20T22:29:12.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:11 smithi078 ceph-mon[52919]: osdmap e110: 8 total, 8 up, 8 in 2024-05-20T22:29:12.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:11 smithi078 ceph-mon[52919]: from='client.44233 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (5m) 6s ago 14m 14.4M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (2m) 6s ago 14m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (2m) 2m ago 14m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (5m) 6s ago 14m 44.5M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (8m) 6s ago 8m 36.9M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (8m) 2m ago 8m 13.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:29:12.236 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (5m) 6s ago 16m 508M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (4m) 2m ago 14m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (4m) 6s ago 16m 49.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (3m) 2m ago 13m 35.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (8m) 6s ago 8m 287M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (8m) 2m ago 8m 87.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (5m) 6s ago 14m 11.4M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (5m) 2m ago 13m 18.4M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (117s) 6s ago 13m 295M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (68s) 6s ago 12m 228M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (17s) 6s ago 12m 12.6M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (11m) 6s ago 11m 389M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (11m) 2m ago 11m 545M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:29:12.237 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (11m) 2m ago 11m 356M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:29:12.238 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (10m) 2m ago 10m 543M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:29:12.238 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (10m) 2m ago 10m 483M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:29:12.238 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (4m) 6s ago 13m 67.9M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:29:12.776 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:29:12.777 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:29:12.778 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:12.778 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:29:12.778 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-05-20T22:29:12.778 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 7 2024-05-20T22:29:12.778 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:29:12.778 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:29:12.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:12 smithi006 ceph-mon[65753]: pgmap v159: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.9 MiB/s wr, 468 op/s; 30/1578 objects degraded (1.901%); 3.9 MiB/s, 44 objects/s recovering 2024-05-20T22:29:12.868 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:12 smithi006 ceph-mon[65753]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:12.868 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:12 smithi006 ceph-mon[65753]: osdmap e111: 8 total, 8 up, 8 in 2024-05-20T22:29:12.868 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:12 smithi006 ceph-mon[65753]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:12.868 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:12 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2268948536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:13.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 219324 10.68 MB/sec execute 379 sec latency 761.247 ms 2024-05-20T22:29:13.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:12 smithi078 ceph-mon[52919]: pgmap v159: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 3.9 MiB/s wr, 468 op/s; 30/1578 objects degraded (1.901%); 3.9 MiB/s, 44 objects/s recovering 2024-05-20T22:29:13.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:12 smithi078 ceph-mon[52919]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:13.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:12 smithi078 ceph-mon[52919]: osdmap e111: 8 total, 8 up, 8 in 2024-05-20T22:29:13.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:12 smithi078 ceph-mon[52919]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:13.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:12 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2268948536' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:13.269 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:29:13.269 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:29:13.269 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "9/23 daemons upgraded", 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:29:13.270 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:29:13.850 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN Degraded data redundancy: 30/1578 objects degraded (1.901%), 2 pgs degraded 2024-05-20T22:29:13.850 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 30/1578 objects degraded (1.901%), 2 pgs degraded 2024-05-20T22:29:13.850 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,6] 2024-05-20T22:29:13.850 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1a is active+recovering+undersized+degraded+remapped, acting [4,1] 2024-05-20T22:29:13.878 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: osdmap e112: 8 total, 8 up, 8 in 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:13.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:13.880 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-05-20T22:29:13.880 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:13.880 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:13 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2967348631' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:29:14.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 219558 10.68 MB/sec execute 380 sec latency 949.993 ms 2024-05-20T22:29:14.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: osdmap e112: 8 total, 8 up, 8 in 2024-05-20T22:29:14.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:14.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:14.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:14.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:13 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2967348631' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:29:14.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:14 smithi006 ceph-mon[65753]: pgmap v162: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.5 MiB/s wr, 450 op/s; 30/1578 objects degraded (1.901%); 3.8 MiB/s, 38 objects/s recovering 2024-05-20T22:29:14.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:14 smithi006 ceph-mon[65753]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:29:15.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 219887 10.69 MB/sec execute 381 sec latency 37.725 ms 2024-05-20T22:29:15.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:14 smithi078 ceph-mon[52919]: pgmap v162: 97 pgs: 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 92 active+clean; 340 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.5 MiB/s wr, 450 op/s; 30/1578 objects degraded (1.901%); 3.8 MiB/s, 38 objects/s recovering 2024-05-20T22:29:15.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:14 smithi078 ceph-mon[52919]: osdmap e113: 8 total, 8 up, 8 in 2024-05-20T22:29:16.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220201 10.68 MB/sec execute 382 sec latency 37.687 ms 2024-05-20T22:29:16.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:16.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:16.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:16 smithi006 ceph-mon[65753]: pgmap v164: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 381 op/s; 9/1584 objects degraded (0.568%); 3.3 MiB/s, 31 objects/s recovering 2024-05-20T22:29:16.701 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:16 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 30/1578 objects degraded (1.901%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:16.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:16.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:16.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:16 smithi078 ceph-mon[52919]: pgmap v164: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 94 active+clean; 337 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 381 op/s; 9/1584 objects degraded (0.568%); 3.3 MiB/s, 31 objects/s recovering 2024-05-20T22:29:16.899 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:16 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 30/1578 objects degraded (1.901%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:17.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220543 10.68 MB/sec execute 383 sec latency 59.119 ms 2024-05-20T22:29:17.733 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:17 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/1584 objects degraded (0.568%), 1 pg degraded) 2024-05-20T22:29:17.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:17 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:29:17.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:17 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/1584 objects degraded (0.568%), 1 pg degraded) 2024-05-20T22:29:17.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:17 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:29:18.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 220772 10.66 MB/sec execute 384 sec latency 40.206 ms 2024-05-20T22:29:18.750 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:18 smithi006 ceph-mon[65753]: pgmap v165: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 MiB/s wr, 358 op/s; 68 KiB/s, 5 objects/s recovering 2024-05-20T22:29:18.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:18 smithi078 ceph-mon[52919]: pgmap v165: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 MiB/s wr, 358 op/s; 68 KiB/s, 5 objects/s recovering 2024-05-20T22:29:19.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 221140 10.65 MB/sec execute 385 sec latency 80.036 ms 2024-05-20T22:29:20.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 221433 10.64 MB/sec execute 386 sec latency 47.214 ms 2024-05-20T22:29:20.844 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:20 smithi006 ceph-mon[65753]: pgmap v166: 97 pgs: 97 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.6 MiB/s wr, 292 op/s; 53 KiB/s, 4 objects/s recovering 2024-05-20T22:29:20.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:20 smithi078 ceph-mon[52919]: pgmap v166: 97 pgs: 97 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 3.6 MiB/s wr, 292 op/s; 53 KiB/s, 4 objects/s recovering 2024-05-20T22:29:21.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 221639 10.62 MB/sec execute 387 sec latency 78.011 ms 2024-05-20T22:29:22.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 221860 10.60 MB/sec execute 388 sec latency 82.304 ms 2024-05-20T22:29:22.712 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:22 smithi006 ceph-mon[65753]: pgmap v167: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 508 op/s; 47 KiB/s, 4 objects/s recovering 2024-05-20T22:29:22.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:22 smithi078 ceph-mon[52919]: pgmap v167: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 508 op/s; 47 KiB/s, 4 objects/s recovering 2024-05-20T22:29:23.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 222141 10.61 MB/sec execute 389 sec latency 51.914 ms 2024-05-20T22:29:24.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 222426 10.60 MB/sec execute 390 sec latency 97.895 ms 2024-05-20T22:29:24.840 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:24 smithi006 ceph-mon[65753]: pgmap v168: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 441 op/s; 41 KiB/s, 3 objects/s recovering 2024-05-20T22:29:25.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 222865 10.60 MB/sec execute 391 sec latency 41.311 ms 2024-05-20T22:29:25.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:24 smithi078 ceph-mon[52919]: pgmap v168: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 441 op/s; 41 KiB/s, 3 objects/s recovering 2024-05-20T22:29:26.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 223234 10.61 MB/sec execute 392 sec latency 41.720 ms 2024-05-20T22:29:26.642 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:26 smithi006 ceph-mon[65753]: pgmap v169: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.9 MiB/s wr, 385 op/s; 35 KiB/s, 2 objects/s recovering 2024-05-20T22:29:26.642 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:26.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:26 smithi078 ceph-mon[52919]: pgmap v169: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.9 MiB/s wr, 385 op/s; 35 KiB/s, 2 objects/s recovering 2024-05-20T22:29:26.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:27.089 INFO:teuthology.orchestra.run.smithi006.stdout: 5 223653 10.62 MB/sec execute 393 sec latency 39.268 ms 2024-05-20T22:29:28.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224068 10.62 MB/sec execute 394 sec latency 54.667 ms 2024-05-20T22:29:28.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:28 smithi006 ceph-mon[65753]: pgmap v170: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.4 MiB/s wr, 489 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:29:28.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:28.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:28 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:28.598 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:28 smithi006 ceph-mon[65753]: Upgrade: osd.3 is safe to restart 2024-05-20T22:29:28.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:28 smithi078 ceph-mon[52919]: pgmap v170: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.4 MiB/s wr, 489 op/s; 0 B/s, 1 objects/s recovering 2024-05-20T22:29:28.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:28.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:28 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-05-20T22:29:28.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:28 smithi078 ceph-mon[52919]: Upgrade: osd.3 is safe to restart 2024-05-20T22:29:29.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224357 10.61 MB/sec execute 395 sec latency 66.205 ms 2024-05-20T22:29:30.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 224752 10.60 MB/sec execute 396 sec latency 40.814 ms 2024-05-20T22:29:30.613 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:30 smithi006 ceph-mon[65753]: pgmap v171: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.1 MiB/s wr, 415 op/s 2024-05-20T22:29:30.614 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:30.614 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:30.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:30 smithi078 ceph-mon[52919]: pgmap v171: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.1 MiB/s wr, 415 op/s 2024-05-20T22:29:30.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:30.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:31.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 225155 10.59 MB/sec execute 397 sec latency 39.158 ms 2024-05-20T22:29:31.898 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:31 smithi006 ceph-mon[65753]: Upgrade: Updating osd.3 2024-05-20T22:29:31.898 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:31.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:29:31.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:31.899 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:31 smithi006 ceph-mon[65753]: Deploying daemon osd.3 on smithi006 2024-05-20T22:29:32.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 225469 10.58 MB/sec execute 398 sec latency 38.135 ms 2024-05-20T22:29:32.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:31 smithi078 ceph-mon[52919]: Upgrade: Updating osd.3 2024-05-20T22:29:32.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:32.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-05-20T22:29:32.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:29:32.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:31 smithi078 ceph-mon[52919]: Deploying daemon osd.3 on smithi006 2024-05-20T22:29:32.910 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:32 smithi006 ceph-mon[65753]: pgmap v172: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.6 MiB/s wr, 560 op/s 2024-05-20T22:29:33.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 225859 10.59 MB/sec execute 399 sec latency 59.383 ms 2024-05-20T22:29:33.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:32 smithi078 ceph-mon[52919]: pgmap v172: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.6 MiB/s wr, 560 op/s 2024-05-20T22:29:34.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226240 10.59 MB/sec execute 400 sec latency 60.306 ms 2024-05-20T22:29:34.983 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:34 smithi006 ceph-mon[65753]: pgmap v173: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 380 op/s 2024-05-20T22:29:35.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226518 10.59 MB/sec execute 401 sec latency 56.835 ms 2024-05-20T22:29:35.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:34 smithi078 ceph-mon[52919]: pgmap v173: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 380 op/s 2024-05-20T22:29:36.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226701 10.58 MB/sec execute 402 sec latency 56.135 ms 2024-05-20T22:29:37.073 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:36 smithi006 ceph-mon[65753]: pgmap v174: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.7 MiB/s wr, 488 op/s 2024-05-20T22:29:37.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 226973 10.58 MB/sec execute 403 sec latency 52.280 ms 2024-05-20T22:29:37.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:36 smithi078 ceph-mon[52919]: pgmap v174: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.7 MiB/s wr, 488 op/s 2024-05-20T22:29:37.583 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:37 smithi006 systemd[1]: Stopping Ceph osd.3 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:29:38.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 227246 10.57 MB/sec execute 404 sec latency 48.964 ms 2024-05-20T22:29:38.092 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:37 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[45688]: 2024-05-20T22:29:37.925+0000 7f7fb0214700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:29:38.092 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:37 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[45688]: 2024-05-20T22:29:37.925+0000 7f7fb0214700 -1 osd.3 113 *** Got signal Terminated *** 2024-05-20T22:29:38.092 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:37 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[45688]: 2024-05-20T22:29:37.925+0000 7f7fb0214700 -1 osd.3 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:29:39.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:38 smithi006 ceph-mon[65753]: pgmap v175: 97 pgs: 97 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-05-20T22:29:39.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:38 smithi006 ceph-mon[65753]: osd.3 marked itself down and dead 2024-05-20T22:29:39.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 227511 10.56 MB/sec execute 405 sec latency 270.222 ms 2024-05-20T22:29:39.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:38 smithi078 ceph-mon[52919]: pgmap v175: 97 pgs: 97 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 614 op/s 2024-05-20T22:29:39.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:38 smithi078 ceph-mon[52919]: osd.3 marked itself down and dead 2024-05-20T22:29:39.785 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:39 smithi006 podman[79267]: 2024-05-20 22:29:39.501633287 +0000 UTC m=+1.848106244 container died 9bedacf00eaa9a69b9f68b46104747d32455150fe6d399625e1e10118b6665cf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, version=8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-type=git, architecture=x86_64) 2024-05-20T22:29:40.073 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:39 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:29:40.073 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:39 smithi006 ceph-mon[65753]: osdmap e114: 8 total, 7 up, 8 in 2024-05-20T22:29:40.074 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:40 smithi006 podman[79267]: 2024-05-20 22:29:40.068390013 +0000 UTC m=+2.414862964 container cleanup 9bedacf00eaa9a69b9f68b46104747d32455150fe6d399625e1e10118b6665cf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, version=8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD) 2024-05-20T22:29:40.074 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:40 smithi006 bash[79267]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3 2024-05-20T22:29:40.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 227547 10.54 MB/sec execute 406 sec latency 1058.510 ms 2024-05-20T22:29:40.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:39 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:29:40.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:39 smithi078 ceph-mon[52919]: osdmap e114: 8 total, 7 up, 8 in 2024-05-20T22:29:40.579 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:40 smithi006 podman[79283]: 2024-05-20 22:29:40.370745916 +0000 UTC m=+0.865709654 container remove 9bedacf00eaa9a69b9f68b46104747d32455150fe6d399625e1e10118b6665cf (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, architecture=x86_64, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vendor=Red Hat, Inc.) 2024-05-20T22:29:40.831 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:40 smithi006 podman[79369]: 2024-05-20 22:29:40.651140576 +0000 UTC m=+0.020122386 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:29:40.831 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:40 smithi006 ceph-mon[65753]: pgmap v177: 97 pgs: 14 peering, 11 stale+active+clean, 72 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-05-20T22:29:40.832 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:40 smithi006 ceph-mon[65753]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-05-20T22:29:40.832 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:40 smithi006 ceph-mon[65753]: osdmap e115: 8 total, 7 up, 8 in 2024-05-20T22:29:41.084 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:40 smithi006 podman[79369]: 2024-05-20 22:29:40.85482862 +0000 UTC m=+0.223810416 container create 520b996592a9786efebbc8a37c8b8e3de147991aaf75612f83e44221697ab805 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:29:41.090 INFO:teuthology.orchestra.run.smithi006.stdout: 5 227853 10.53 MB/sec execute 407 sec latency 42.023 ms 2024-05-20T22:29:41.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:40 smithi078 ceph-mon[52919]: pgmap v177: 97 pgs: 14 peering, 11 stale+active+clean, 72 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.9 MiB/s wr, 478 op/s 2024-05-20T22:29:41.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:40 smithi078 ceph-mon[52919]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-05-20T22:29:41.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:40 smithi078 ceph-mon[52919]: osdmap e115: 8 total, 7 up, 8 in 2024-05-20T22:29:41.340 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:41 smithi006 podman[79369]: 2024-05-20 22:29:41.318764728 +0000 UTC m=+0.687746533 container init 520b996592a9786efebbc8a37c8b8e3de147991aaf75612f83e44221697ab805 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:29:41.341 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:41 smithi006 podman[79369]: 2024-05-20 22:29:41.328964121 +0000 UTC m=+0.697945926 container start 520b996592a9786efebbc8a37c8b8e3de147991aaf75612f83e44221697ab805 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-20T22:29:41.592 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:41 smithi006 podman[79369]: 2024-05-20 22:29:41.501840262 +0000 UTC m=+0.870822063 container attach 520b996592a9786efebbc8a37c8b8e3de147991aaf75612f83e44221697ab805 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:29:42.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 228193 10.52 MB/sec execute 408 sec latency 47.166 ms 2024-05-20T22:29:42.094 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:41 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 153/1617 objects degraded (9.462%), 18 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:42.094 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:41 smithi006 podman[79369]: 2024-05-20 22:29:41.848301167 +0000 UTC m=+1.217282966 container died 520b996592a9786efebbc8a37c8b8e3de147991aaf75612f83e44221697ab805 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5) 2024-05-20T22:29:42.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:41 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 153/1617 objects degraded (9.462%), 18 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:43.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 228578 10.51 MB/sec execute 409 sec latency 53.331 ms 2024-05-20T22:29:43.122 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:42 smithi006 podman[79394]: 2024-05-20 22:29:42.952113657 +0000 UTC m=+1.092153306 container remove 520b996592a9786efebbc8a37c8b8e3de147991aaf75612f83e44221697ab805 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:29:43.122 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:42 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.3.service: Deactivated successfully. 2024-05-20T22:29:43.122 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:42 smithi006 systemd[1]: Stopped Ceph osd.3 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:29:43.122 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:42 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.3.service: Consumed 4min 34.961s CPU time. 2024-05-20T22:29:43.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:42 smithi006 ceph-mon[65753]: pgmap v179: 97 pgs: 18 active+undersized+degraded, 14 peering, 9 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 575 op/s; 153/1617 objects degraded (9.462%) 2024-05-20T22:29:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:42 smithi078 ceph-mon[52919]: pgmap v179: 97 pgs: 18 active+undersized+degraded, 14 peering, 9 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 575 op/s; 153/1617 objects degraded (9.462%) 2024-05-20T22:29:43.373 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:43 smithi006 systemd[1]: Starting Ceph osd.3 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:29:43.644 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:43 smithi006 podman[79488]: 2024-05-20 22:29:43.510195621 +0000 UTC m=+0.026829667 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:29:43.899 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:43 smithi006 podman[79488]: 2024-05-20 22:29:43.653690012 +0000 UTC m=+0.170324041 container create 6168e29c09d12ab1ca64e6b7a83990795213b4b9d2bbe3e3fd6551f938832c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-20T22:29:44.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 228851 10.50 MB/sec execute 410 sec latency 66.743 ms 2024-05-20T22:29:44.156 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 podman[79488]: 2024-05-20 22:29:44.091444803 +0000 UTC m=+0.608078858 container init 6168e29c09d12ab1ca64e6b7a83990795213b4b9d2bbe3e3fd6551f938832c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS) 2024-05-20T22:29:44.156 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 podman[79488]: 2024-05-20 22:29:44.100654818 +0000 UTC m=+0.617288855 container start 6168e29c09d12ab1ca64e6b7a83990795213b4b9d2bbe3e3fd6551f938832c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514) 2024-05-20T22:29:44.410 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 podman[79488]: 2024-05-20 22:29:44.266962869 +0000 UTC m=+0.783596905 container attach 6168e29c09d12ab1ca64e6b7a83990795213b4b9d2bbe3e3fd6551f938832c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:29:44.422 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:29:44.928 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:44 smithi006 ceph-mon[65753]: pgmap v180: 97 pgs: 18 active+undersized+degraded, 14 peering, 9 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 413 op/s; 153/1617 objects degraded (9.462%) 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate[79500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate[79500]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 bash[79488]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 bash[79488]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 bash[79488]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 bash[79488]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 bash[79488]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 bash[79488]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 bash[79488]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate[79500]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate[79500]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T22:29:44.929 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate[79500]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-05-20T22:29:44.930 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate[79500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-05-20T22:29:44.930 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate[79500]: --> ceph-volume raw activate successful for osd ID: 3 2024-05-20T22:29:44.930 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:44 smithi006 podman[79488]: 2024-05-20 22:29:44.841752887 +0000 UTC m=+1.358386926 container died 6168e29c09d12ab1ca64e6b7a83990795213b4b9d2bbe3e3fd6551f938832c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0) 2024-05-20T22:29:45.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 229159 10.50 MB/sec execute 411 sec latency 49.024 ms 2024-05-20T22:29:45.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:44 smithi078 ceph-mon[52919]: pgmap v180: 97 pgs: 18 active+undersized+degraded, 14 peering, 9 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 413 op/s; 153/1617 objects degraded (9.462%) 2024-05-20T22:29:45.537 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (6m) 39s ago 15m 14.4M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (3m) 39s ago 15m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (3m) 2m ago 14m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 39s ago 15m 44.5M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (9m) 39s ago 9m 36.9M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (9m) 2m ago 9m 13.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (5m) 39s ago 16m 508M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (5m) 2m ago 14m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (4m) 39s ago 16m 49.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (4m) 2m ago 14m 35.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:29:45.538 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (9m) 39s ago 9m 287M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (8m) 2m ago 8m 87.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (6m) 39s ago 14m 11.4M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 2m ago 14m 18.4M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (2m) 39s ago 13m 295M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (101s) 39s ago 13m 228M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (51s) 39s ago 12m 12.6M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (12m) 39s ago 12m 389M 1937M 17.2.0 e1d6a67b021e 9bedacf00eaa 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (12m) 2m ago 12m 545M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (11m) 2m ago 11m 356M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (11m) 2m ago 11m 543M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:29:45.539 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (10m) 2m ago 10m 483M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:29:45.540 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (5m) 39s ago 14m 67.9M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:29:45.850 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:45 smithi006 podman[79685]: 2024-05-20 22:29:45.783436548 +0000 UTC m=+0.932977156 container remove 6168e29c09d12ab1ca64e6b7a83990795213b4b9d2bbe3e3fd6551f938832c56 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-activate, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-05-20T22:29:45.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:45 smithi006 ceph-mon[65753]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:45.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:45 smithi006 ceph-mon[65753]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:45.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:45.942 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:46.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 229590 10.50 MB/sec execute 412 sec latency 37.610 ms 2024-05-20T22:29:46.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:45 smithi078 ceph-mon[52919]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:46.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:45 smithi078 ceph-mon[52919]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:46.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:46.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:29:46.192 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:46 smithi006 podman[79780]: 2024-05-20 22:29:45.980446973 +0000 UTC m=+0.028534525 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:29:46.192 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:46 smithi006 podman[79780]: 2024-05-20 22:29:46.099765944 +0000 UTC m=+0.147853487 container create b90d07d0f5f4dfc6023f13bd7c8acc251d1a61ce6becc6b680847eb57a5f674f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:46.203 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 3 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 7 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:29:46.204 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:29:46.443 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:46 smithi006 podman[79780]: 2024-05-20 22:29:46.411714994 +0000 UTC m=+0.459802551 container init b90d07d0f5f4dfc6023f13bd7c8acc251d1a61ce6becc6b680847eb57a5f674f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-05-20T22:29:46.443 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:46 smithi006 podman[79780]: 2024-05-20 22:29:46.425283992 +0000 UTC m=+0.473371532 container start b90d07d0f5f4dfc6023f13bd7c8acc251d1a61ce6becc6b680847eb57a5f674f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux ) 2024-05-20T22:29:46.694 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:46 smithi006 ceph-osd[79822]: -- 172.21.15.6:0/2944544444 <== mon.0 v2:172.21.15.6:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b6ac907680 con 0x55b6ac8d1000 2024-05-20T22:29:46.694 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:46 smithi006 bash[79780]: b90d07d0f5f4dfc6023f13bd7c8acc251d1a61ce6becc6b680847eb57a5f674f 2024-05-20T22:29:46.695 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:46 smithi006 systemd[1]: Started Ceph osd.3 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:29:46.767 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:29:46.767 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:29:46.767 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:29:46.767 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "9/23 daemons upgraded", 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:29:46.768 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:29:46.946 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:46 smithi006 ceph-mon[65753]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:46.946 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:46 smithi006 ceph-mon[65753]: pgmap v181: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.1 MiB/s wr, 358 op/s; 199/1596 objects degraded (12.469%) 2024-05-20T22:29:46.946 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:46 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1192201560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:46.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:46 smithi006 ceph-mon[65753]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-05-20T22:29:46.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:46.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:46.947 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:47.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 230014 10.50 MB/sec execute 413 sec latency 29.981 ms 2024-05-20T22:29:47.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:46 smithi078 ceph-mon[52919]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:47.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:46 smithi078 ceph-mon[52919]: pgmap v181: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.1 MiB/s wr, 358 op/s; 199/1596 objects degraded (12.469%) 2024-05-20T22:29:47.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:46 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1192201560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:29:47.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:46 smithi078 ceph-mon[52919]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-05-20T22:29:47.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:47.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:47.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:29:47.202 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:47 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[79795]: 2024-05-20T22:29:47.007+0000 7f69894fe7c0 -1 Falling back to public interface 2024-05-20T22:29:47.416 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 199/1596 objects degraded (12.469%), 27 pgs degraded 2024-05-20T22:29:47.416 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-05-20T22:29:47.416 INFO:teuthology.orchestra.run.smithi006.stdout: osd.3 (root=default,host=smithi006) is down 2024-05-20T22:29:47.416 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 199/1596 objects degraded (12.469%), 27 pgs degraded 2024-05-20T22:29:47.416 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.12 is active+undersized+degraded, acting [5,2] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-05-20T22:29:47.417 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-05-20T22:29:47.418 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-05-20T22:29:47.959 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:47 smithi006 ceph-mon[65753]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:47.959 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:47 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3032283953' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:29:48.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 230366 10.52 MB/sec execute 414 sec latency 38.287 ms 2024-05-20T22:29:48.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:47 smithi078 ceph-mon[52919]: from='client.44271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:29:48.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:47 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3032283953' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:29:48.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:48 smithi006 ceph-mon[65753]: pgmap v182: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 420 op/s; 215/1602 objects degraded (13.421%) 2024-05-20T22:29:48.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:48 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 215/1602 objects degraded (13.421%), 27 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:49.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 230760 10.52 MB/sec execute 415 sec latency 35.958 ms 2024-05-20T22:29:49.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:48 smithi078 ceph-mon[52919]: pgmap v182: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 337 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 420 op/s; 215/1602 objects degraded (13.421%) 2024-05-20T22:29:49.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:48 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 215/1602 objects degraded (13.421%), 27 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:50.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 231080 10.51 MB/sec execute 416 sec latency 38.859 ms 2024-05-20T22:29:51.015 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:50 smithi006 ceph-mon[65753]: pgmap v183: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 534 op/s; 216/1635 objects degraded (13.211%) 2024-05-20T22:29:51.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 231298 10.50 MB/sec execute 417 sec latency 50.466 ms 2024-05-20T22:29:51.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:50 smithi078 ceph-mon[52919]: pgmap v183: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 345 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 534 op/s; 216/1635 objects degraded (13.211%) 2024-05-20T22:29:52.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 231599 10.50 MB/sec execute 418 sec latency 61.344 ms 2024-05-20T22:29:53.047 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:52 smithi006 ceph-mon[65753]: pgmap v184: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.6 MiB/s wr, 526 op/s; 214/1665 objects degraded (12.853%) 2024-05-20T22:29:53.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 232022 10.48 MB/sec execute 419 sec latency 37.872 ms 2024-05-20T22:29:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:52 smithi078 ceph-mon[52919]: pgmap v184: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.6 MiB/s wr, 526 op/s; 214/1665 objects degraded (12.853%) 2024-05-20T22:29:54.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 232342 10.47 MB/sec execute 420 sec latency 50.735 ms 2024-05-20T22:29:54.560 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:54 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[79795]: 2024-05-20T22:29:54.321+0000 7f69894fe7c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-05-20T22:29:55.068 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:54 smithi006 ceph-mon[65753]: pgmap v185: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 375 op/s; 214/1665 objects degraded (12.853%) 2024-05-20T22:29:55.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 232718 10.48 MB/sec execute 421 sec latency 39.416 ms 2024-05-20T22:29:55.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:54 smithi078 ceph-mon[52919]: pgmap v185: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 346 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 375 op/s; 214/1665 objects degraded (12.853%) 2024-05-20T22:29:56.086 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:55 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 214/1665 objects degraded (12.853%), 27 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:56.086 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:55 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[79795]: 2024-05-20T22:29:55.933+0000 7f69894fe7c0 -1 osd.3 113 log_to_monitors true 2024-05-20T22:29:56.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 233082 10.47 MB/sec execute 422 sec latency 52.127 ms 2024-05-20T22:29:56.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:55 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 214/1665 objects degraded (12.853%), 27 pgs degraded (PG_DEGRADED) 2024-05-20T22:29:57.092 INFO:teuthology.orchestra.run.smithi006.stdout: 5 233559 10.48 MB/sec execute 423 sec latency 42.841 ms 2024-05-20T22:29:57.101 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:29:56 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[79795]: 2024-05-20T22:29:56.966+0000 7f697a955700 -1 osd.3 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:29:57.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:56 smithi006 ceph-mon[65753]: pgmap v186: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 214/1641 objects degraded (13.041%) 2024-05-20T22:29:57.101 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:56 smithi006 ceph-mon[65753]: from='osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:57.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:56 smithi078 ceph-mon[52919]: pgmap v186: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 342 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 4.5 MiB/s wr, 443 op/s; 214/1641 objects degraded (13.041%) 2024-05-20T22:29:57.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:56 smithi078 ceph-mon[52919]: from='osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-05-20T22:29:58.092 INFO:teuthology.orchestra.run.smithi006.stdout: 5 233875 10.49 MB/sec execute 424 sec latency 138.749 ms 2024-05-20T22:29:58.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:57 smithi006 ceph-mon[65753]: from='osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T22:29:58.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:57 smithi006 ceph-mon[65753]: osdmap e116: 8 total, 7 up, 8 in 2024-05-20T22:29:58.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:57 smithi006 ceph-mon[65753]: from='osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:29:58.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:57 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:58.123 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:57 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:58.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:57 smithi078 ceph-mon[52919]: from='osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-05-20T22:29:58.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:57 smithi078 ceph-mon[52919]: osdmap e116: 8 total, 7 up, 8 in 2024-05-20T22:29:58.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:57 smithi078 ceph-mon[52919]: from='osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi006", "root=default"]}]: dispatch 2024-05-20T22:29:58.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:57 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:58.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:57 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:59.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 234043 10.47 MB/sec execute 425 sec latency 1138.782 ms 2024-05-20T22:29:59.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:58 smithi006 ceph-mon[65753]: pgmap v188: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 589 op/s; 217/1623 objects degraded (13.370%) 2024-05-20T22:29:59.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:58 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:29:59.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:58 smithi006 ceph-mon[65753]: osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069] boot 2024-05-20T22:29:59.163 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:58 smithi006 ceph-mon[65753]: osdmap e117: 8 total, 8 up, 8 in 2024-05-20T22:29:59.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:58 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:59.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:58 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:59.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:58 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:58 smithi078 ceph-mon[52919]: pgmap v188: 97 pgs: 27 active+undersized+degraded, 14 active+undersized, 56 active+clean; 340 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.5 MiB/s wr, 589 op/s; 217/1623 objects degraded (13.370%) 2024-05-20T22:29:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:58 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:29:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:58 smithi078 ceph-mon[52919]: osd.3 [v2:172.21.15.6:6826/3753244069,v1:172.21.15.6:6827/3753244069] boot 2024-05-20T22:29:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:58 smithi078 ceph-mon[52919]: osdmap e117: 8 total, 8 up, 8 in 2024-05-20T22:29:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:58 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-05-20T22:29:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:58 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:29:59.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:58 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:00.092 INFO:teuthology.orchestra.run.smithi006.stdout: 5 234216 10.45 MB/sec execute 426 sec latency 2138.810 ms 2024-05-20T22:30:00.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:29:59 smithi006 ceph-mon[65753]: osdmap e118: 8 total, 8 up, 8 in 2024-05-20T22:30:00.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:29:59 smithi078 ceph-mon[52919]: osdmap e118: 8 total, 8 up, 8 in 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pgmap v191: 97 pgs: 3 remapped+peering, 19 active+undersized+degraded, 1 active+recovery_wait+degraded, 11 active+undersized, 63 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 577 op/s; 177/1632 objects degraded (10.846%); 1.7 KiB/s, 0 objects/s recovering 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: osdmap e119: 8 total, 8 up, 8 in 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: Health detail: HEALTH_WARN Degraded data redundancy: 177/1632 objects degraded (10.846%), 20 pgs degraded 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: [WRN] PG_DEGRADED: Degraded data redundancy: 177/1632 objects degraded (10.846%), 20 pgs degraded 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 2.12 is active+undersized+degraded, acting [5,2] 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-05-20T22:30:00.972 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.c is active+undersized+degraded, acting [5,6] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 3.19 is active+recovery_wait+degraded, acting [1,3,4] 2024-05-20T22:30:00.973 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-05-20T22:30:00.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-05-20T22:30:00.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-05-20T22:30:00.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:00.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:00.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 177/1632 objects degraded (10.846%), 20 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:00.974 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:00 smithi006 ceph-mon[65753]: osdmap e120: 8 total, 8 up, 8 in 2024-05-20T22:30:01.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 234473 10.45 MB/sec execute 427 sec latency 2426.707 ms 2024-05-20T22:30:01.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pgmap v191: 97 pgs: 3 remapped+peering, 19 active+undersized+degraded, 1 active+recovery_wait+degraded, 11 active+undersized, 63 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 577 op/s; 177/1632 objects degraded (10.846%); 1.7 KiB/s, 0 objects/s recovering 2024-05-20T22:30:01.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: osdmap e119: 8 total, 8 up, 8 in 2024-05-20T22:30:01.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: Health detail: HEALTH_WARN Degraded data redundancy: 177/1632 objects degraded (10.846%), 20 pgs degraded 2024-05-20T22:30:01.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: [WRN] PG_DEGRADED: Degraded data redundancy: 177/1632 objects degraded (10.846%), 20 pgs degraded 2024-05-20T22:30:01.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 2.12 is active+undersized+degraded, acting [5,2] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.b is active+undersized+degraded, acting [0,4] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.c is active+undersized+degraded, acting [5,6] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-05-20T22:30:01.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 3.19 is active+recovery_wait+degraded, acting [1,3,4] 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 4.14 is active+undersized+degraded, acting [2,7] 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 177/1632 objects degraded (10.846%), 20 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:01.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:00 smithi078 ceph-mon[52919]: osdmap e120: 8 total, 8 up, 8 in 2024-05-20T22:30:02.092 INFO:teuthology.orchestra.run.smithi006.stdout: 5 234919 10.46 MB/sec execute 428 sec latency 321.932 ms 2024-05-20T22:30:02.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:02 smithi006 ceph-mon[65753]: pgmap v194: 97 pgs: 1 peering, 76 active+clean, 4 active+undersized, 1 active+recovery_wait+degraded, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+undersized+remapped; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 6.1 MiB/s wr, 526 op/s; 97/1629 objects degraded (5.955%); 72 KiB/s, 36 objects/s recovering 2024-05-20T22:30:02.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:02 smithi006 ceph-mon[65753]: osdmap e121: 8 total, 8 up, 8 in 2024-05-20T22:30:02.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:02 smithi078 ceph-mon[52919]: pgmap v194: 97 pgs: 1 peering, 76 active+clean, 4 active+undersized, 1 active+recovery_wait+degraded, 3 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+undersized+remapped; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 6.1 MiB/s wr, 526 op/s; 97/1629 objects degraded (5.955%); 72 KiB/s, 36 objects/s recovering 2024-05-20T22:30:02.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:02 smithi078 ceph-mon[52919]: osdmap e121: 8 total, 8 up, 8 in 2024-05-20T22:30:03.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 235070 10.45 MB/sec execute 429 sec latency 890.301 ms 2024-05-20T22:30:03.641 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:03 smithi006 ceph-mon[65753]: osdmap e122: 8 total, 8 up, 8 in 2024-05-20T22:30:03.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:03 smithi078 ceph-mon[52919]: osdmap e122: 8 total, 8 up, 8 in 2024-05-20T22:30:04.092 INFO:teuthology.orchestra.run.smithi006.stdout: 5 235474 10.43 MB/sec execute 430 sec latency 30.923 ms 2024-05-20T22:30:05.092 INFO:teuthology.orchestra.run.smithi006.stdout: 5 235794 10.42 MB/sec execute 431 sec latency 58.250 ms 2024-05-20T22:30:05.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: pgmap v197: 97 pgs: 5 peering, 88 active+clean, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s wr, 505 op/s; 42/1653 objects degraded (2.541%); 679 KiB/s, 57 objects/s recovering 2024-05-20T22:30:05.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:05.312 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:05.313 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:05 smithi006 ceph-mon[65753]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-20T22:30:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: pgmap v197: 97 pgs: 5 peering, 88 active+clean, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 MiB/s wr, 505 op/s; 42/1653 objects degraded (2.541%); 679 KiB/s, 57 objects/s recovering 2024-05-20T22:30:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:05 smithi078 ceph-mon[52919]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-05-20T22:30:06.075 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:06 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 42/1653 objects degraded (2.541%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:06.091 INFO:teuthology.orchestra.run.smithi006.stdout: 5 236249 10.43 MB/sec execute 432 sec latency 32.926 ms 2024-05-20T22:30:06.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:06 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 42/1653 objects degraded (2.541%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:07.092 INFO:teuthology.orchestra.run.smithi006.stdout: 5 236685 10.45 MB/sec execute 433 sec latency 48.374 ms 2024-05-20T22:30:07.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:07 smithi006 ceph-mon[65753]: pgmap v198: 97 pgs: 5 peering, 90 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 523 op/s; 24/1653 objects degraded (1.452%); 1.2 MiB/s, 44 objects/s recovering 2024-05-20T22:30:07.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:07 smithi078 ceph-mon[52919]: pgmap v198: 97 pgs: 5 peering, 90 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 523 op/s; 24/1653 objects degraded (1.452%); 1.2 MiB/s, 44 objects/s recovering 2024-05-20T22:30:08.098 INFO:teuthology.orchestra.run.smithi006.stdout: 5 237072 10.44 MB/sec execute 434 sec latency 42.386 ms 2024-05-20T22:30:08.354 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:08 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1653 objects degraded (1.452%), 1 pg degraded) 2024-05-20T22:30:08.354 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:08 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:30:08.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:08 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/1653 objects degraded (1.452%), 1 pg degraded) 2024-05-20T22:30:08.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:08 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:30:09.098 INFO:teuthology.orchestra.run.smithi006.stdout: 5 237393 10.43 MB/sec execute 435 sec latency 52.713 ms 2024-05-20T22:30:09.364 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:09 smithi006 ceph-mon[65753]: pgmap v199: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 6.4 MiB/s wr, 555 op/s; 1.1 MiB/s, 42 objects/s recovering 2024-05-20T22:30:09.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:09 smithi078 ceph-mon[52919]: pgmap v199: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 6.4 MiB/s wr, 555 op/s; 1.1 MiB/s, 42 objects/s recovering 2024-05-20T22:30:10.098 INFO:teuthology.orchestra.run.smithi006.stdout: 5 237802 10.44 MB/sec execute 436 sec latency 50.919 ms 2024-05-20T22:30:11.098 INFO:teuthology.orchestra.run.smithi006.stdout: 5 238123 10.44 MB/sec execute 437 sec latency 54.744 ms 2024-05-20T22:30:11.148 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:11 smithi006 ceph-mon[65753]: pgmap v200: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.8 MiB/s wr, 423 op/s; 963 KiB/s, 19 objects/s recovering 2024-05-20T22:30:11.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:11 smithi078 ceph-mon[52919]: pgmap v200: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.8 MiB/s wr, 423 op/s; 963 KiB/s, 19 objects/s recovering 2024-05-20T22:30:12.101 INFO:teuthology.orchestra.run.smithi006.stdout: 5 238488 10.44 MB/sec execute 438 sec latency 44.000 ms 2024-05-20T22:30:13.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 238818 10.43 MB/sec execute 439 sec latency 52.242 ms 2024-05-20T22:30:13.214 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:13 smithi006 ceph-mon[65753]: pgmap v201: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.5 MiB/s wr, 409 op/s; 777 KiB/s, 15 objects/s recovering 2024-05-20T22:30:13.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:13 smithi078 ceph-mon[52919]: pgmap v201: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.5 MiB/s wr, 409 op/s; 777 KiB/s, 15 objects/s recovering 2024-05-20T22:30:14.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 239118 10.41 MB/sec execute 440 sec latency 78.470 ms 2024-05-20T22:30:15.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 239324 10.40 MB/sec execute 441 sec latency 64.955 ms 2024-05-20T22:30:15.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:15 smithi006 ceph-mon[65753]: pgmap v202: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 478 op/s; 482 KiB/s, 5 objects/s recovering 2024-05-20T22:30:15.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:15 smithi078 ceph-mon[52919]: pgmap v202: 97 pgs: 97 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 478 op/s; 482 KiB/s, 5 objects/s recovering 2024-05-20T22:30:16.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 239610 10.40 MB/sec execute 442 sec latency 77.371 ms 2024-05-20T22:30:16.590 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:16.590 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:16.590 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:16 smithi006 ceph-mon[65753]: pgmap v203: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 4.9 MiB/s wr, 524 op/s; 439 KiB/s, 5 objects/s recovering 2024-05-20T22:30:16.591 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:16.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:16.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:16.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:16 smithi078 ceph-mon[52919]: pgmap v203: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 4.9 MiB/s wr, 524 op/s; 439 KiB/s, 5 objects/s recovering 2024-05-20T22:30:16.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:17.101 INFO:teuthology.orchestra.run.smithi006.stdout: 5 239897 10.39 MB/sec execute 443 sec latency 47.959 ms 2024-05-20T22:30:18.055 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:30:18.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 240180 10.40 MB/sec execute 444 sec latency 47.173 ms 2024-05-20T22:30:18.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:18 smithi006 ceph-mon[65753]: pgmap v204: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 547 op/s; 78 KiB/s, 3 objects/s recovering 2024-05-20T22:30:18.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:18 smithi006 ceph-mon[65753]: from='client.34354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:18.853 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:30:18.853 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (7m) 21s ago 15m 14.7M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:30:18.853 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (3m) 21s ago 15m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:30:18.853 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (3m) 3m ago 15m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (6m) 21s ago 15m 43.8M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (9m) 21s ago 9m 37.6M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (9m) 3m ago 9m 13.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (6m) 21s ago 17m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (5m) 3m ago 15m 451M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (5m) 21s ago 17m 52.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (4m) 3m ago 15m 35.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (9m) 21s ago 9m 283M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (9m) 3m ago 9m 87.5M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (6m) 21s ago 15m 11.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (6m) 3m ago 14m 18.4M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (3m) 21s ago 14m 358M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:30:18.854 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (2m) 21s ago 13m 287M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:30:18.855 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (84s) 21s ago 13m 196M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:30:18.855 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (32s) 21s ago 12m 11.8M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:30:18.855 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (12m) 3m ago 12m 545M 2704M 17.2.0 e1d6a67b021e c8b773e5d432 2024-05-20T22:30:18.855 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (12m) 3m ago 12m 356M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:30:18.855 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (11m) 3m ago 11m 543M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:30:18.855 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (11m) 3m ago 11m 483M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:30:18.855 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (5m) 21s ago 14m 68.0M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:30:18.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:18 smithi078 ceph-mon[52919]: pgmap v204: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 547 op/s; 78 KiB/s, 3 objects/s recovering 2024-05-20T22:30:18.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:18 smithi078 ceph-mon[52919]: from='client.34354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:19.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 240394 10.38 MB/sec execute 445 sec latency 104.709 ms 2024-05-20T22:30:19.441 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:30:19.441 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:30:19.441 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:30:19.441 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:19.441 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:30:19.441 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:30:19.441 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 4 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:30:19.442 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:30:19.721 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:19 smithi006 ceph-mon[65753]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:19.721 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:19 smithi006 ceph-mon[65753]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:19.721 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:19 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:19.721 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:19 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:19.721 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:19 smithi006 ceph-mon[65753]: Upgrade: osd.4 is safe to restart 2024-05-20T22:30:19.721 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:19 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1689701846' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:19.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:19 smithi078 ceph-mon[52919]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:19.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:19 smithi078 ceph-mon[52919]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:19.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:19 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:19.892 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:19 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-05-20T22:30:19.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:19 smithi078 ceph-mon[52919]: Upgrade: osd.4 is safe to restart 2024-05-20T22:30:19.893 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:19 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1689701846' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:20.078 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:30:20.078 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:30:20.078 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "10/23 daemons upgraded", 2024-05-20T22:30:20.079 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:30:20.080 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:30:20.080 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:30:20.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 240767 10.38 MB/sec execute 446 sec latency 37.917 ms 2024-05-20T22:30:20.721 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:30:20.740 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:20 smithi006 ceph-mon[65753]: pgmap v205: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.4 MiB/s wr, 480 op/s 2024-05-20T22:30:20.740 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:20 smithi006 ceph-mon[65753]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:20.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:20 smithi078 ceph-mon[52919]: pgmap v205: 97 pgs: 97 active+clean; 336 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.4 MiB/s wr, 480 op/s 2024-05-20T22:30:20.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:20 smithi078 ceph-mon[52919]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:21.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 241014 10.37 MB/sec execute 447 sec latency 73.870 ms 2024-05-20T22:30:21.751 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:21 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2586052821' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:30:21.751 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:21 smithi006 ceph-mon[65753]: Upgrade: Updating osd.4 2024-05-20T22:30:21.751 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:21.751 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:30:21.751 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:21.752 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:21 smithi006 ceph-mon[65753]: Deploying daemon osd.4 on smithi078 2024-05-20T22:30:21.881 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:21 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2586052821' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:30:21.881 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:21 smithi078 ceph-mon[52919]: Upgrade: Updating osd.4 2024-05-20T22:30:21.881 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:21.881 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-05-20T22:30:21.881 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:21.881 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:21 smithi078 ceph-mon[52919]: Deploying daemon osd.4 on smithi078 2024-05-20T22:30:22.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 241384 10.38 MB/sec execute 448 sec latency 45.433 ms 2024-05-20T22:30:23.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 241848 10.39 MB/sec execute 449 sec latency 28.664 ms 2024-05-20T22:30:23.277 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:23 smithi006 ceph-mon[65753]: pgmap v206: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 450 op/s 2024-05-20T22:30:23.347 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:23 smithi078 ceph-mon[52919]: pgmap v206: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 450 op/s 2024-05-20T22:30:24.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 242320 10.38 MB/sec execute 450 sec latency 38.903 ms 2024-05-20T22:30:25.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 242687 10.37 MB/sec execute 451 sec latency 66.174 ms 2024-05-20T22:30:25.304 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:25 smithi006 ceph-mon[65753]: pgmap v207: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-05-20T22:30:25.396 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:25 smithi078 systemd[1]: Stopping Ceph osd.4 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:30:25.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:25 smithi078 ceph-mon[52919]: pgmap v207: 97 pgs: 97 active+clean; 338 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-05-20T22:30:25.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:25 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[31426]: 2024-05-20T22:30:25.466+0000 7f6a17109700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:30:25.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:25 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[31426]: 2024-05-20T22:30:25.466+0000 7f6a17109700 -1 osd.4 122 *** Got signal Terminated *** 2024-05-20T22:30:25.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:25 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[31426]: 2024-05-20T22:30:25.466+0000 7f6a17109700 -1 osd.4 122 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:30:26.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 243033 10.38 MB/sec execute 452 sec latency 71.805 ms 2024-05-20T22:30:26.316 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:26 smithi006 ceph-mon[65753]: osd.4 marked itself down and dead 2024-05-20T22:30:26.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:26 smithi078 ceph-mon[52919]: osd.4 marked itself down and dead 2024-05-20T22:30:26.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:26 smithi078 podman[57378]: 2024-05-20 22:30:26.706532958 +0000 UTC m=+1.367726892 container died c8b773e5d43230b51e8da0e1f0d5c7fdee2dbf6555ad48609560a4190943b4c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, release=754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, version=8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-20T22:30:27.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 243121 10.36 MB/sec execute 453 sec latency 1008.310 ms 2024-05-20T22:30:27.352 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:27 smithi006 ceph-mon[65753]: pgmap v208: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.8 MiB/s wr, 403 op/s 2024-05-20T22:30:27.352 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:27 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:30:27.352 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:27 smithi006 ceph-mon[65753]: osdmap e123: 8 total, 7 up, 8 in 2024-05-20T22:30:27.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:27 smithi078 ceph-mon[52919]: pgmap v208: 97 pgs: 97 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.8 MiB/s wr, 403 op/s 2024-05-20T22:30:27.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:27 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:30:27.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:27 smithi078 ceph-mon[52919]: osdmap e123: 8 total, 7 up, 8 in 2024-05-20T22:30:27.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:27 smithi078 podman[57378]: 2024-05-20 22:30:27.45804929 +0000 UTC m=+2.119243221 container cleanup c8b773e5d43230b51e8da0e1f0d5c7fdee2dbf6555ad48609560a4190943b4c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-05-20T22:30:27.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:27 smithi078 bash[57378]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4 2024-05-20T22:30:27.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:27 smithi078 podman[57405]: 2024-05-20 22:30:27.651378334 +0000 UTC m=+0.945553869 container remove c8b773e5d43230b51e8da0e1f0d5c7fdee2dbf6555ad48609560a4190943b4c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, version=8, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., distribution-scope=public, build-date=2022-05-03T08:36:31.336870) 2024-05-20T22:30:28.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 243500 10.35 MB/sec execute 454 sec latency 1021.749 ms 2024-05-20T22:30:28.112 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:28 smithi006 ceph-mon[65753]: osdmap e124: 8 total, 7 up, 8 in 2024-05-20T22:30:28.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:28 smithi078 ceph-mon[52919]: osdmap e124: 8 total, 7 up, 8 in 2024-05-20T22:30:28.396 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:28 smithi078 podman[57493]: 2024-05-20 22:30:28.01917954 +0000 UTC m=+0.018429034 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:30:28.396 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:28 smithi078 podman[57493]: 2024-05-20 22:30:28.152412689 +0000 UTC m=+0.151662175 container create 72e7818f842808929de3568e1f57581423daf3d9826b58ac40801e3803addb69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-20T22:30:28.764 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:28 smithi078 podman[57493]: 2024-05-20 22:30:28.432475675 +0000 UTC m=+0.431725183 container init 72e7818f842808929de3568e1f57581423daf3d9826b58ac40801e3803addb69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T22:30:28.764 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:28 smithi078 podman[57493]: 2024-05-20 22:30:28.435794199 +0000 UTC m=+0.435043695 container start 72e7818f842808929de3568e1f57581423daf3d9826b58ac40801e3803addb69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True) 2024-05-20T22:30:28.764 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:28 smithi078 podman[57493]: 2024-05-20 22:30:28.538568729 +0000 UTC m=+0.537818227 container attach 72e7818f842808929de3568e1f57581423daf3d9826b58ac40801e3803addb69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:30:28.764 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:28 smithi078 podman[57493]: 2024-05-20 22:30:28.769160375 +0000 UTC m=+0.768409870 container died 72e7818f842808929de3568e1f57581423daf3d9826b58ac40801e3803addb69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-05-20T22:30:29.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 243860 10.36 MB/sec execute 455 sec latency 49.555 ms 2024-05-20T22:30:29.120 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:29 smithi006 ceph-mon[65753]: pgmap v211: 97 pgs: 20 peering, 77 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 548 op/s 2024-05-20T22:30:29.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:29 smithi078 ceph-mon[52919]: pgmap v211: 97 pgs: 20 peering, 77 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 548 op/s 2024-05-20T22:30:29.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:29 smithi078 podman[57519]: 2024-05-20 22:30:29.547431149 +0000 UTC m=+0.766291505 container remove 72e7818f842808929de3568e1f57581423daf3d9826b58ac40801e3803addb69 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, ceph=True) 2024-05-20T22:30:29.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:29 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.4.service: Deactivated successfully. 2024-05-20T22:30:29.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:29 smithi078 systemd[1]: Stopped Ceph osd.4 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:30:29.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:29 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.4.service: Consumed 4min 37.543s CPU time. 2024-05-20T22:30:29.896 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:29 smithi078 systemd[1]: Starting Ceph osd.4 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:30:30.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 244240 10.36 MB/sec execute 456 sec latency 36.348 ms 2024-05-20T22:30:30.366 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:30 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 49/1638 objects degraded (2.991%), 5 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:30.366 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 podman[57648]: 2024-05-20 22:30:29.923677956 +0000 UTC m=+0.020501139 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:30:30.366 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 podman[57648]: 2024-05-20 22:30:30.244966698 +0000 UTC m=+0.341789882 container create e28638027409e115d078087638c0950d703deaa7f7649bcffbf1e6673d537272 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=) 2024-05-20T22:30:30.396 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:30 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 49/1638 objects degraded (2.991%), 5 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:30.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 podman[57648]: 2024-05-20 22:30:30.472651919 +0000 UTC m=+0.569475086 container init e28638027409e115d078087638c0950d703deaa7f7649bcffbf1e6673d537272 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True) 2024-05-20T22:30:30.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 podman[57648]: 2024-05-20 22:30:30.476972222 +0000 UTC m=+0.573795407 container start e28638027409e115d078087638c0950d703deaa7f7649bcffbf1e6673d537272 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-05-20T22:30:30.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 podman[57648]: 2024-05-20 22:30:30.572917627 +0000 UTC m=+0.669740806 container attach e28638027409e115d078087638c0950d703deaa7f7649bcffbf1e6673d537272 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-05-20T22:30:31.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 244505 10.35 MB/sec execute 457 sec latency 49.500 ms 2024-05-20T22:30:31.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:31 smithi006 ceph-mon[65753]: pgmap v212: 97 pgs: 4 active+undersized, 5 active+undersized+degraded, 20 peering, 68 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 582 op/s; 49/1638 objects degraded (2.991%) 2024-05-20T22:30:31.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:31.164 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:31.305 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:31 smithi078 ceph-mon[52919]: pgmap v212: 97 pgs: 4 active+undersized, 5 active+undersized+degraded, 20 peering, 68 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 582 op/s; 49/1638 objects degraded (2.991%) 2024-05-20T22:30:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:31.306 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:31.306 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate[57662]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T22:30:31.306 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate[57662]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T22:30:31.306 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 bash[57648]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T22:30:31.306 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 bash[57648]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-05-20T22:30:31.306 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 bash[57648]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 bash[57648]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 bash[57648]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 bash[57648]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 bash[57648]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate[57662]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate[57662]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate[57662]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate[57662]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate[57662]: --> ceph-volume raw activate successful for osd ID: 4 2024-05-20T22:30:31.307 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:30 smithi078 podman[57648]: 2024-05-20 22:30:30.998160897 +0000 UTC m=+1.094984085 container died e28638027409e115d078087638c0950d703deaa7f7649bcffbf1e6673d537272 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:30:31.990 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:31 smithi078 podman[57777]: 2024-05-20 22:30:31.744686581 +0000 UTC m=+0.740588721 container remove e28638027409e115d078087638c0950d703deaa7f7649bcffbf1e6673d537272 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-activate, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-20T22:30:31.991 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:31 smithi078 podman[57838]: 2024-05-20 22:30:31.866442556 +0000 UTC m=+0.017566566 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:30:31.991 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:31 smithi078 podman[57838]: 2024-05-20 22:30:31.995613307 +0000 UTC m=+0.146737295 container create c332350301f25a0a0f54a3ff29267131d84f6a275ae270599ef2a1c4ecbaec75 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:30:32.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 244847 10.35 MB/sec execute 458 sec latency 46.953 ms 2024-05-20T22:30:32.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:32 smithi078 podman[57838]: 2024-05-20 22:30:32.257078838 +0000 UTC m=+0.408202823 container init c332350301f25a0a0f54a3ff29267131d84f6a275ae270599ef2a1c4ecbaec75 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-20T22:30:32.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:32 smithi078 podman[57838]: 2024-05-20 22:30:32.261003594 +0000 UTC m=+0.412127588 container start c332350301f25a0a0f54a3ff29267131d84f6a275ae270599ef2a1c4ecbaec75 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-20T22:30:32.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:32 smithi078 ceph-osd[57857]: -- 172.21.15.78:0/608373602 <== mon.1 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560be556d680 con 0x560be5536c00 2024-05-20T22:30:32.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:32 smithi078 bash[57838]: c332350301f25a0a0f54a3ff29267131d84f6a275ae270599ef2a1c4ecbaec75 2024-05-20T22:30:32.646 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:32 smithi078 systemd[1]: Started Ceph osd.4 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:30:33.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 245074 10.34 MB/sec execute 459 sec latency 53.621 ms 2024-05-20T22:30:33.146 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:32 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:30:32.808+0000 7f7d570287c0 -1 Falling back to public interface 2024-05-20T22:30:33.212 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:33 smithi006 ceph-mon[65753]: pgmap v213: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 634 op/s; 253/1659 objects degraded (15.250%) 2024-05-20T22:30:33.212 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:33.212 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:33.212 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:33 smithi078 ceph-mon[52919]: pgmap v213: 97 pgs: 9 active+undersized, 27 active+undersized+degraded, 5 peering, 56 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.2 MiB/s wr, 634 op/s; 253/1659 objects degraded (15.250%) 2024-05-20T22:30:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:33.561 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:34.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 245408 10.35 MB/sec execute 460 sec latency 41.248 ms 2024-05-20T22:30:35.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 245697 10.34 MB/sec execute 461 sec latency 72.176 ms 2024-05-20T22:30:35.236 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:35 smithi006 ceph-mon[65753]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.7 MiB/s wr, 504 op/s; 270/1662 objects degraded (16.245%) 2024-05-20T22:30:35.426 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:35 smithi078 ceph-mon[52919]: pgmap v214: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.7 MiB/s wr, 504 op/s; 270/1662 objects degraded (16.245%) 2024-05-20T22:30:36.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246002 10.33 MB/sec execute 462 sec latency 45.968 ms 2024-05-20T22:30:36.263 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:36 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 270/1662 objects degraded (16.245%), 28 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:36.570 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:36 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 270/1662 objects degraded (16.245%), 28 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:37.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246254 10.31 MB/sec execute 463 sec latency 46.286 ms 2024-05-20T22:30:37.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:37 smithi006 ceph-mon[65753]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 509 op/s; 266/1638 objects degraded (16.239%) 2024-05-20T22:30:37.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:37.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:37.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:37 smithi078 ceph-mon[52919]: pgmap v215: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 509 op/s; 266/1638 objects degraded (16.239%) 2024-05-20T22:30:37.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:37.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:38.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246580 10.32 MB/sec execute 464 sec latency 45.686 ms 2024-05-20T22:30:39.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 246861 10.31 MB/sec execute 465 sec latency 57.204 ms 2024-05-20T22:30:39.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:39 smithi006 ceph-mon[65753]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 475 op/s; 272/1644 objects degraded (16.545%) 2024-05-20T22:30:39.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:39.299 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:39.439 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:39 smithi078 ceph-mon[52919]: pgmap v216: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 475 op/s; 272/1644 objects degraded (16.545%) 2024-05-20T22:30:39.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:39.440 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:40.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247215 10.31 MB/sec execute 466 sec latency 45.051 ms 2024-05-20T22:30:40.547 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:40 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:30:40.353+0000 7f7d570287c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-05-20T22:30:41.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247430 10.30 MB/sec execute 467 sec latency 101.975 ms 2024-05-20T22:30:41.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:41 smithi006 ceph-mon[65753]: pgmap v217: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 425 op/s; 271/1635 objects degraded (16.575%) 2024-05-20T22:30:41.372 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:41 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 271/1635 objects degraded (16.575%), 28 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:41.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:41 smithi078 ceph-mon[52919]: pgmap v217: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 425 op/s; 271/1635 objects degraded (16.575%) 2024-05-20T22:30:41.592 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:41 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 271/1635 objects degraded (16.575%), 28 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:42.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247700 10.29 MB/sec execute 468 sec latency 47.668 ms 2024-05-20T22:30:42.146 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:41 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:30:41.941+0000 7f7d570287c0 -1 osd.4 122 log_to_monitors true 2024-05-20T22:30:42.424 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:42 smithi006 ceph-mon[65753]: from='osd.4 [v2:172.21.15.78:6800/2095274899,v1:172.21.15.78:6801/2095274899]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:30:42.424 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:42 smithi006 ceph-mon[65753]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:30:42.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:42 smithi078 ceph-mon[52919]: from='osd.4 [v2:172.21.15.78:6800/2095274899,v1:172.21.15.78:6801/2095274899]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:30:42.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:42 smithi078 ceph-mon[52919]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-05-20T22:30:43.073 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:30:42 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:30:42.702+0000 7f7d44477700 -1 osd.4 122 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:30:43.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 247909 10.28 MB/sec execute 469 sec latency 215.216 ms 2024-05-20T22:30:43.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: pgmap v218: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 262/1587 objects degraded (16.509%) 2024-05-20T22:30:43.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T22:30:43.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: osdmap e125: 8 total, 7 up, 8 in 2024-05-20T22:30:43.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='osd.4 [v2:172.21.15.78:6800/2095274899,v1:172.21.15.78:6801/2095274899]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:30:43.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:30:43.217 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='osd.4 ' entity='osd.4' 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:43.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: pgmap v218: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 262/1587 objects degraded (16.509%) 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: osdmap e125: 8 total, 7 up, 8 in 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='osd.4 [v2:172.21.15.78:6800/2095274899,v1:172.21.15.78:6801/2095274899]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='osd.4 ' entity='osd.4' 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:43.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:44.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248143 10.27 MB/sec execute 470 sec latency 401.307 ms 2024-05-20T22:30:44.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:44 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:44.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:44 smithi006 ceph-mon[65753]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-05-20T22:30:44.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:44 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:30:44.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:44 smithi006 ceph-mon[65753]: osd.4 [v2:172.21.15.78:6800/2095274899,v1:172.21.15.78:6801/2095274899] boot 2024-05-20T22:30:44.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:44 smithi006 ceph-mon[65753]: osdmap e126: 8 total, 8 up, 8 in 2024-05-20T22:30:44.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:44 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:30:44.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:44 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:44.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:44 smithi078 ceph-mon[52919]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-05-20T22:30:44.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:44 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:30:44.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:44 smithi078 ceph-mon[52919]: osd.4 [v2:172.21.15.78:6800/2095274899,v1:172.21.15.78:6801/2095274899] boot 2024-05-20T22:30:44.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:44 smithi078 ceph-mon[52919]: osdmap e126: 8 total, 8 up, 8 in 2024-05-20T22:30:44.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:44 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-05-20T22:30:45.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248186 10.25 MB/sec execute 471 sec latency 1401.340 ms 2024-05-20T22:30:45.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:45 smithi006 ceph-mon[65753]: pgmap v220: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 504 op/s; 258/1599 objects degraded (16.135%) 2024-05-20T22:30:45.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:45 smithi006 ceph-mon[65753]: osdmap e127: 8 total, 8 up, 8 in 2024-05-20T22:30:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:45 smithi078 ceph-mon[52919]: pgmap v220: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 504 op/s; 258/1599 objects degraded (16.135%) 2024-05-20T22:30:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:45 smithi078 ceph-mon[52919]: osdmap e127: 8 total, 8 up, 8 in 2024-05-20T22:30:46.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248295 10.24 MB/sec execute 472 sec latency 2073.372 ms 2024-05-20T22:30:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:46 smithi006 ceph-mon[65753]: pgmap v223: 97 pgs: 5 peering, 11 active+undersized, 25 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 354 op/s; 232/1611 objects degraded (14.401%) 2024-05-20T22:30:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:46 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 258/1599 objects degraded (16.135%), 28 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:46 smithi006 ceph-mon[65753]: osdmap e128: 8 total, 8 up, 8 in 2024-05-20T22:30:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:30:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:30:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:46 smithi078 ceph-mon[52919]: pgmap v223: 97 pgs: 5 peering, 11 active+undersized, 25 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 354 op/s; 232/1611 objects degraded (14.401%) 2024-05-20T22:30:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:46 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 258/1599 objects degraded (16.135%), 28 pgs degraded (PG_DEGRADED) 2024-05-20T22:30:46.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:46 smithi078 ceph-mon[52919]: osdmap e128: 8 total, 8 up, 8 in 2024-05-20T22:30:47.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248630 10.24 MB/sec execute 473 sec latency 377.045 ms 2024-05-20T22:30:47.814 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:47 smithi006 ceph-mon[65753]: osdmap e129: 8 total, 8 up, 8 in 2024-05-20T22:30:48.102 INFO:teuthology.orchestra.run.smithi006.stdout: 5 248809 10.24 MB/sec execute 474 sec latency 1014.360 ms 2024-05-20T22:30:48.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:47 smithi078 ceph-mon[52919]: osdmap e129: 8 total, 8 up, 8 in 2024-05-20T22:30:48.833 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:48 smithi006 ceph-mon[65753]: pgmap v226: 97 pgs: 1 active+recovery_wait+degraded, 8 peering, 6 active+undersized+remapped, 82 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.6 MiB/s wr, 284 op/s; 13/1533 objects degraded (0.848%); 2.3 MiB/s, 73 objects/s recovering 2024-05-20T22:30:48.833 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:48 smithi006 ceph-mon[65753]: osdmap e130: 8 total, 8 up, 8 in 2024-05-20T22:30:49.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249083 10.22 MB/sec execute 475 sec latency 60.018 ms 2024-05-20T22:30:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:48 smithi078 ceph-mon[52919]: pgmap v226: 97 pgs: 1 active+recovery_wait+degraded, 8 peering, 6 active+undersized+remapped, 82 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.6 MiB/s wr, 284 op/s; 13/1533 objects degraded (0.848%); 2.3 MiB/s, 73 objects/s recovering 2024-05-20T22:30:49.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:48 smithi078 ceph-mon[52919]: osdmap e130: 8 total, 8 up, 8 in 2024-05-20T22:30:50.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249362 10.22 MB/sec execute 476 sec latency 70.236 ms 2024-05-20T22:30:50.924 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:50 smithi006 ceph-mon[65753]: pgmap v228: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 6 active+undersized+remapped, 87 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 436 op/s; 13/1515 objects degraded (0.858%); 2.1 MiB/s, 53 keys/s, 65 objects/s recovering 2024-05-20T22:30:50.924 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:50 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 13/1515 objects degraded (0.858%), 1 pg degraded (PG_DEGRADED) 2024-05-20T22:30:51.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249556 10.20 MB/sec execute 477 sec latency 72.270 ms 2024-05-20T22:30:51.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:50 smithi078 ceph-mon[52919]: pgmap v228: 97 pgs: 1 active+recovery_wait+degraded, 3 peering, 6 active+undersized+remapped, 87 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 436 op/s; 13/1515 objects degraded (0.858%); 2.1 MiB/s, 53 keys/s, 65 objects/s recovering 2024-05-20T22:30:51.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:50 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 13/1515 objects degraded (0.858%), 1 pg degraded (PG_DEGRADED) 2024-05-20T22:30:51.304 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:30:51.955 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:51 smithi006 ceph-mon[65753]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:52.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249755 10.19 MB/sec execute 478 sec latency 52.498 ms 2024-05-20T22:30:52.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:51 smithi078 ceph-mon[52919]: from='client.44303 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:52.393 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:30:52.393 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (7m) 55s ago 16m 14.7M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:30:52.393 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (4m) 55s ago 16m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (4m) 15s ago 15m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (7m) 55s ago 16m 43.8M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (10m) 55s ago 10m 37.6M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (10m) 15s ago 10m 13.9M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (7m) 55s ago 17m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (6m) 15s ago 15m 453M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (5m) 55s ago 17m 52.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (5m) 15s ago 15m 41.8M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (10m) 55s ago 10m 283M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (10m) 15s ago 10m 88.0M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 55s ago 16m 11.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:30:52.394 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 15s ago 15m 16.7M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (3m) 55s ago 14m 358M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (2m) 55s ago 14m 287M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (117s) 55s ago 14m 196M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (66s) 55s ago 13m 11.8M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (20s) 15s ago 13m 12.6M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (12m) 15s ago 12m 571M 2704M 17.2.0 e1d6a67b021e 4734b000ab7b 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (12m) 15s ago 12m 496M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (11m) 15s ago 11m 576M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:30:52.395 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 55s ago 15m 68.0M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:30:52.745 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:52 smithi006 ceph-mon[65753]: pgmap v229: 97 pgs: 3 peering, 94 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 1.8 MiB/s, 43 keys/s, 54 objects/s recovering 2024-05-20T22:30:52.745 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:52 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1515 objects degraded (0.858%), 1 pg degraded) 2024-05-20T22:30:52.745 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:52 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:30:52.745 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:52 smithi006 ceph-mon[65753]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:53.077 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:30:53.077 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:30:53.077 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:30:53.077 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:53.077 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:30:53.077 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 5 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 9 2024-05-20T22:30:53.078 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:30:53.079 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:30:53.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 249963 10.18 MB/sec execute 479 sec latency 60.453 ms 2024-05-20T22:30:53.143 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:52 smithi078 ceph-mon[52919]: pgmap v229: 97 pgs: 3 peering, 94 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.2 MiB/s wr, 412 op/s; 1.8 MiB/s, 43 keys/s, 54 objects/s recovering 2024-05-20T22:30:53.143 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:52 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1515 objects degraded (0.858%), 1 pg degraded) 2024-05-20T22:30:53.143 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:52 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:30:53.143 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:52 smithi078 ceph-mon[52919]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "11/23 daemons upgraded", 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:30:53.624 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:30:53.625 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:30:53.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:53 smithi006 ceph-mon[65753]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:53.804 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:53 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3545862694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:54.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 250193 10.17 MB/sec execute 480 sec latency 72.036 ms 2024-05-20T22:30:54.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:53 smithi078 ceph-mon[52919]: from='client.44307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:54.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:53 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3545862694' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:30:54.316 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:30:54.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:54 smithi006 ceph-mon[65753]: pgmap v230: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.6 MiB/s wr, 338 op/s; 1.4 MiB/s, 33 keys/s, 42 objects/s recovering 2024-05-20T22:30:54.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:54 smithi006 ceph-mon[65753]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:54.813 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:54 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2926550140' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:30:55.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 250546 10.17 MB/sec execute 481 sec latency 41.020 ms 2024-05-20T22:30:55.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:54 smithi078 ceph-mon[52919]: pgmap v230: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.6 MiB/s wr, 338 op/s; 1.4 MiB/s, 33 keys/s, 42 objects/s recovering 2024-05-20T22:30:55.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:54 smithi078 ceph-mon[52919]: from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:30:55.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:54 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2926550140' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:30:56.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 250980 10.18 MB/sec execute 482 sec latency 37.603 ms 2024-05-20T22:30:56.851 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:56 smithi006 ceph-mon[65753]: pgmap v231: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.8 MiB/s wr, 328 op/s; 273 KiB/s, 29 keys/s, 14 objects/s recovering 2024-05-20T22:30:57.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 251482 10.18 MB/sec execute 483 sec latency 29.874 ms 2024-05-20T22:30:57.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:56 smithi078 ceph-mon[52919]: pgmap v231: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.8 MiB/s wr, 328 op/s; 273 KiB/s, 29 keys/s, 14 objects/s recovering 2024-05-20T22:30:58.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 251879 10.18 MB/sec execute 484 sec latency 45.043 ms 2024-05-20T22:30:58.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:58 smithi006 ceph-mon[65753]: pgmap v232: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 423 op/s; 129 KiB/s, 26 keys/s, 3 objects/s recovering 2024-05-20T22:30:58.884 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:58 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:58.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:58 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:58.885 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:30:58 smithi006 ceph-mon[65753]: Upgrade: osd.5 is safe to restart 2024-05-20T22:30:59.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 252136 10.18 MB/sec execute 485 sec latency 55.417 ms 2024-05-20T22:30:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:58 smithi078 ceph-mon[52919]: pgmap v232: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 423 op/s; 129 KiB/s, 26 keys/s, 3 objects/s recovering 2024-05-20T22:30:59.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:58 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:59.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:58 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-05-20T22:30:59.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:30:58 smithi078 ceph-mon[52919]: Upgrade: osd.5 is safe to restart 2024-05-20T22:31:00.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 252461 10.18 MB/sec execute 486 sec latency 48.876 ms 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: pgmap v233: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 425 op/s; 109 KiB/s, 22 keys/s, 3 objects/s recovering 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: Upgrade: Updating osd.5 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: Deploying daemon osd.5 on smithi078 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:00.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: pgmap v233: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 425 op/s; 109 KiB/s, 22 keys/s, 3 objects/s recovering 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: Upgrade: Updating osd.5 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: Deploying daemon osd.5 on smithi078 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:00.941 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:01.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 252788 10.17 MB/sec execute 487 sec latency 64.544 ms 2024-05-20T22:31:02.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253092 10.17 MB/sec execute 488 sec latency 64.230 ms 2024-05-20T22:31:02.734 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:02 smithi006 ceph-mon[65753]: pgmap v234: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 407 op/s; 23 KiB/s, 1 objects/s recovering 2024-05-20T22:31:02.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:02 smithi078 ceph-mon[52919]: pgmap v234: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 407 op/s; 23 KiB/s, 1 objects/s recovering 2024-05-20T22:31:03.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253319 10.16 MB/sec execute 489 sec latency 50.228 ms 2024-05-20T22:31:03.627 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:03 smithi078 systemd[1]: Stopping Ceph osd.5 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:31:04.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253585 10.15 MB/sec execute 490 sec latency 69.652 ms 2024-05-20T22:31:04.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[34389]: 2024-05-20T22:31:03.957+0000 7f25c324a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:31:04.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[34389]: 2024-05-20T22:31:03.957+0000 7f25c324a700 -1 osd.5 130 *** Got signal Terminated *** 2024-05-20T22:31:04.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[34389]: 2024-05-20T22:31:03.957+0000 7f25c324a700 -1 osd.5 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:31:05.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:04 smithi006 ceph-mon[65753]: pgmap v235: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 397 op/s 2024-05-20T22:31:05.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:04 smithi006 ceph-mon[65753]: osd.5 marked itself down and dead 2024-05-20T22:31:05.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253822 10.15 MB/sec execute 491 sec latency 426.625 ms 2024-05-20T22:31:05.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:04 smithi078 ceph-mon[52919]: pgmap v235: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 397 op/s 2024-05-20T22:31:05.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:04 smithi078 ceph-mon[52919]: osd.5 marked itself down and dead 2024-05-20T22:31:05.611 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:05 smithi078 podman[59630]: 2024-05-20 22:31:05.342581221 +0000 UTC m=+1.795477261 container died 4734b000ab7b9f66c157a08c18cb35dabfdf65143f11e93be00be8f13cb39fc2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, io.openshift.tags=base centos centos-stream, architecture=x86_64, ceph=True, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, version=8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0) 2024-05-20T22:31:05.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:05 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:31:05.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:05 smithi078 ceph-mon[52919]: osdmap e131: 8 total, 7 up, 8 in 2024-05-20T22:31:05.896 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:05 smithi078 podman[59630]: 2024-05-20 22:31:05.715200877 +0000 UTC m=+2.168096912 container cleanup 4734b000ab7b9f66c157a08c18cb35dabfdf65143f11e93be00be8f13cb39fc2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., architecture=x86_64, vcs-type=git, release=754, io.openshift.tags=base centos centos-stream, ceph=True, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-05-20T22:31:05.896 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:05 smithi078 bash[59630]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5 2024-05-20T22:31:05.936 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:05 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:31:05.937 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:05 smithi006 ceph-mon[65753]: osdmap e131: 8 total, 7 up, 8 in 2024-05-20T22:31:06.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 253923 10.13 MB/sec execute 492 sec latency 1094.338 ms 2024-05-20T22:31:06.395 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:06 smithi078 podman[59647]: 2024-05-20 22:31:06.016671941 +0000 UTC m=+0.667976027 container remove 4734b000ab7b9f66c157a08c18cb35dabfdf65143f11e93be00be8f13cb39fc2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, version=8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, ceph=True, io.buildah.version=1.19.8, distribution-scope=public, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, release=754, io.openshift.expose-services=, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-05-20T22:31:06.695 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:06 smithi078 podman[59736]: 2024-05-20 22:31:06.306381442 +0000 UTC m=+0.018924450 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:31:06.695 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:06 smithi078 podman[59736]: 2024-05-20 22:31:06.451114419 +0000 UTC m=+0.163657476 container create 2a8ce281984dd2c8b6823abcfe7234c2d9d026d32ac4baa75f8c032a2fcbbf9a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-05-20T22:31:06.704 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:06 smithi006 ceph-mon[65753]: pgmap v237: 97 pgs: 12 stale+active+clean, 85 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 504 op/s 2024-05-20T22:31:06.705 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:06 smithi006 ceph-mon[65753]: osdmap e132: 8 total, 7 up, 8 in 2024-05-20T22:31:07.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:06 smithi078 ceph-mon[52919]: pgmap v237: 97 pgs: 12 stale+active+clean, 85 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.9 MiB/s wr, 504 op/s 2024-05-20T22:31:07.058 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:06 smithi078 ceph-mon[52919]: osdmap e132: 8 total, 7 up, 8 in 2024-05-20T22:31:07.058 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:06 smithi078 podman[59736]: 2024-05-20 22:31:06.712516738 +0000 UTC m=+0.425059744 container init 2a8ce281984dd2c8b6823abcfe7234c2d9d026d32ac4baa75f8c032a2fcbbf9a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True) 2024-05-20T22:31:07.058 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:06 smithi078 podman[59736]: 2024-05-20 22:31:06.716184841 +0000 UTC m=+0.428727850 container start 2a8ce281984dd2c8b6823abcfe7234c2d9d026d32ac4baa75f8c032a2fcbbf9a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514) 2024-05-20T22:31:07.059 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:06 smithi078 podman[59736]: 2024-05-20 22:31:06.821075411 +0000 UTC m=+0.533618430 container attach 2a8ce281984dd2c8b6823abcfe7234c2d9d026d32ac4baa75f8c032a2fcbbf9a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-20T22:31:07.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 254277 10.13 MB/sec execute 493 sec latency 38.763 ms 2024-05-20T22:31:07.395 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:07 smithi078 podman[59736]: 2024-05-20 22:31:07.063508664 +0000 UTC m=+0.776051674 container died 2a8ce281984dd2c8b6823abcfe7234c2d9d026d32ac4baa75f8c032a2fcbbf9a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-05-20T22:31:07.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:07 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 152/1599 objects degraded (9.506%), 18 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:07.896 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:07 smithi078 podman[59761]: 2024-05-20 22:31:07.741955358 +0000 UTC m=+0.669877136 container remove 2a8ce281984dd2c8b6823abcfe7234c2d9d026d32ac4baa75f8c032a2fcbbf9a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-20T22:31:07.896 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:07 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.5.service: Deactivated successfully. 2024-05-20T22:31:07.897 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:07 smithi078 systemd[1]: Stopped Ceph osd.5 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:31:07.897 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:07 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.5.service: Consumed 5min 5.442s CPU time. 2024-05-20T22:31:08.002 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:07 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 152/1599 objects degraded (9.506%), 18 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:08.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 254655 10.13 MB/sec execute 494 sec latency 41.250 ms 2024-05-20T22:31:08.246 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:07 smithi078 systemd[1]: Starting Ceph osd.5 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:31:08.246 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:08 smithi078 podman[59857]: 2024-05-20 22:31:08.129455164 +0000 UTC m=+0.018659625 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:31:08.516 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:08 smithi078 podman[59857]: 2024-05-20 22:31:08.251662932 +0000 UTC m=+0.140867372 container create 1d3fdafc54b12894a6e021acfb70bce92e7fd5bdaaaaf0e37d79bf48464a6852 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-20T22:31:08.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:08 smithi006 ceph-mon[65753]: pgmap v239: 97 pgs: 6 active+undersized, 18 active+undersized+degraded, 2 stale+active+clean, 71 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 455 op/s; 152/1599 objects degraded (9.506%) 2024-05-20T22:31:08.896 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:08 smithi078 podman[59857]: 2024-05-20 22:31:08.521281479 +0000 UTC m=+0.410485916 container init 1d3fdafc54b12894a6e021acfb70bce92e7fd5bdaaaaf0e37d79bf48464a6852 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-05-20T22:31:08.896 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:08 smithi078 podman[59857]: 2024-05-20 22:31:08.524830196 +0000 UTC m=+0.414034646 container start 1d3fdafc54b12894a6e021acfb70bce92e7fd5bdaaaaf0e37d79bf48464a6852 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-05-20T22:31:08.896 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:08 smithi078 podman[59857]: 2024-05-20 22:31:08.621350176 +0000 UTC m=+0.510554636 container attach 1d3fdafc54b12894a6e021acfb70bce92e7fd5bdaaaaf0e37d79bf48464a6852 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, ceph=True) 2024-05-20T22:31:08.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:08 smithi078 ceph-mon[52919]: pgmap v239: 97 pgs: 6 active+undersized, 18 active+undersized+degraded, 2 stale+active+clean, 71 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.9 MiB/s wr, 455 op/s; 152/1599 objects degraded (9.506%) 2024-05-20T22:31:09.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255045 10.13 MB/sec execute 495 sec latency 37.133 ms 2024-05-20T22:31:09.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate[59872]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T22:31:09.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate[59872]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T22:31:09.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 bash[59857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T22:31:09.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 bash[59857]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-05-20T22:31:09.396 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 bash[59857]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 bash[59857]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 bash[59857]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 bash[59857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 bash[59857]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate[59872]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate[59872]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate[59872]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate[59872]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate[59872]: --> ceph-volume raw activate successful for osd ID: 5 2024-05-20T22:31:09.397 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:09 smithi078 podman[59857]: 2024-05-20 22:31:09.066143547 +0000 UTC m=+0.955347995 container died 1d3fdafc54b12894a6e021acfb70bce92e7fd5bdaaaaf0e37d79bf48464a6852 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5) 2024-05-20T22:31:10.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255476 10.13 MB/sec execute 496 sec latency 44.711 ms 2024-05-20T22:31:10.380 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 podman[59987]: 2024-05-20 22:31:10.134316092 +0000 UTC m=+1.062213250 container remove 1d3fdafc54b12894a6e021acfb70bce92e7fd5bdaaaaf0e37d79bf48464a6852 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-05-20T22:31:10.380 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 podman[60080]: 2024-05-20 22:31:10.255415742 +0000 UTC m=+0.018816949 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:31:10.380 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 podman[60080]: 2024-05-20 22:31:10.385385959 +0000 UTC m=+0.148787169 container create ff01021e2b383c4cc7d1945999beb3b9292717198d0df1e83b0c145af64d2015 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True) 2024-05-20T22:31:10.645 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 podman[60080]: 2024-05-20 22:31:10.629754378 +0000 UTC m=+0.393155598 container init ff01021e2b383c4cc7d1945999beb3b9292717198d0df1e83b0c145af64d2015 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:31:10.646 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 podman[60080]: 2024-05-20 22:31:10.632973219 +0000 UTC m=+0.396374427 container start ff01021e2b383c4cc7d1945999beb3b9292717198d0df1e83b0c145af64d2015 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:31:10.786 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:10 smithi006 ceph-mon[65753]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.5 MiB/s wr, 357 op/s; 178/1611 objects degraded (11.049%) 2024-05-20T22:31:11.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 255814 10.14 MB/sec execute 497 sec latency 51.538 ms 2024-05-20T22:31:11.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:10 smithi078 ceph-mon[52919]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.5 MiB/s wr, 357 op/s; 178/1611 objects degraded (11.049%) 2024-05-20T22:31:11.146 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 ceph-osd[60099]: -- 172.21.15.78:0/283679271 <== mon.1 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5637b0a1b680 con 0x5637b09e5000 2024-05-20T22:31:11.146 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 bash[60080]: ff01021e2b383c4cc7d1945999beb3b9292717198d0df1e83b0c145af64d2015 2024-05-20T22:31:11.146 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:10 smithi078 systemd[1]: Started Ceph osd.5 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:31:11.646 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:11 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:31:11.178+0000 7f402c2b47c0 -1 Falling back to public interface 2024-05-20T22:31:12.066 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:12.066 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:12.066 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:12.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 256253 10.14 MB/sec execute 498 sec latency 35.223 ms 2024-05-20T22:31:12.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:12.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:12.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:12.832 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:12 smithi006 ceph-mon[65753]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 597 op/s; 179/1638 objects degraded (10.928%) 2024-05-20T22:31:13.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 256698 10.14 MB/sec execute 499 sec latency 37.860 ms 2024-05-20T22:31:13.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:12 smithi078 ceph-mon[52919]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 597 op/s; 179/1638 objects degraded (10.928%) 2024-05-20T22:31:14.099 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:13 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 175/1632 objects degraded (10.723%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:14.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 257109 10.14 MB/sec execute 500 sec latency 58.619 ms 2024-05-20T22:31:14.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:13 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 175/1632 objects degraded (10.723%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:15.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 257531 10.14 MB/sec execute 501 sec latency 37.761 ms 2024-05-20T22:31:15.110 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:14 smithi006 ceph-mon[65753]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 428 op/s; 175/1632 objects degraded (10.723%) 2024-05-20T22:31:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:14 smithi078 ceph-mon[52919]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.8 MiB/s wr, 428 op/s; 175/1632 objects degraded (10.723%) 2024-05-20T22:31:16.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 257878 10.15 MB/sec execute 502 sec latency 54.988 ms 2024-05-20T22:31:16.132 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:16.133 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:16.133 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:16.133 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:16.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:16.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:16.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:16.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:17.103 INFO:teuthology.orchestra.run.smithi006.stdout: 5 258182 10.14 MB/sec execute 503 sec latency 48.779 ms 2024-05-20T22:31:17.108 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:17 smithi006 ceph-mon[65753]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 406 op/s; 179/1650 objects degraded (10.848%) 2024-05-20T22:31:17.109 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:17.109 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:17.327 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:17 smithi078 ceph-mon[52919]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 406 op/s; 179/1650 objects degraded (10.848%) 2024-05-20T22:31:17.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:17.328 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:18.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 258559 10.14 MB/sec execute 504 sec latency 43.487 ms 2024-05-20T22:31:19.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 258961 10.14 MB/sec execute 505 sec latency 36.360 ms 2024-05-20T22:31:19.121 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:19 smithi078 ceph-mon[52919]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 508 op/s; 184/1656 objects degraded (11.111%) 2024-05-20T22:31:19.148 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:19 smithi006 ceph-mon[65753]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 508 op/s; 184/1656 objects degraded (11.111%) 2024-05-20T22:31:19.758 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:31:19.488+0000 7f402c2b47c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-05-20T22:31:20.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 259244 10.13 MB/sec execute 506 sec latency 52.326 ms 2024-05-20T22:31:20.166 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:20 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 179/1626 objects degraded (11.009%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:20.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:20 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 179/1626 objects degraded (11.009%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:21.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 259644 10.14 MB/sec execute 507 sec latency 62.113 ms 2024-05-20T22:31:21.121 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:31:20.826+0000 7f402c2b47c0 -1 osd.5 130 log_to_monitors true 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 570 op/s; 179/1626 objects degraded (11.009%) 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.202 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:21.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:21.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:21.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:21.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:21.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='osd.5 [v2:172.21.15.78:6808/2845769459,v1:172.21.15.78:6809/2845769459]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:31:21.203 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:21 smithi006 ceph-mon[65753]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:31:21.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.3 MiB/s wr, 570 op/s; 179/1626 objects degraded (11.009%) 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:21.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:21.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='osd.5 [v2:172.21.15.78:6808/2845769459,v1:172.21.15.78:6809/2845769459]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:31:21.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:21 smithi078 ceph-mon[52919]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-05-20T22:31:22.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260100 10.14 MB/sec execute 508 sec latency 48.106 ms 2024-05-20T22:31:22.123 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:31:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:31:21.703+0000 7f401d70b700 -1 osd.5 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:31:22.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:22 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:22.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:22 smithi006 ceph-mon[65753]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-05-20T22:31:22.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:22 smithi006 ceph-mon[65753]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T22:31:22.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:22 smithi006 ceph-mon[65753]: osdmap e133: 8 total, 7 up, 8 in 2024-05-20T22:31:22.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:22 smithi006 ceph-mon[65753]: from='osd.5 [v2:172.21.15.78:6808/2845769459,v1:172.21.15.78:6809/2845769459]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:31:22.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:22 smithi006 ceph-mon[65753]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:31:22.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:22 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:22.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:22 smithi078 ceph-mon[52919]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-05-20T22:31:22.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:22 smithi078 ceph-mon[52919]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-05-20T22:31:22.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:22 smithi078 ceph-mon[52919]: osdmap e133: 8 total, 7 up, 8 in 2024-05-20T22:31:22.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:22 smithi078 ceph-mon[52919]: from='osd.5 [v2:172.21.15.78:6808/2845769459,v1:172.21.15.78:6809/2845769459]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:31:22.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:22 smithi078 ceph-mon[52919]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:31:23.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260307 10.13 MB/sec execute 509 sec latency 432.817 ms 2024-05-20T22:31:23.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:23 smithi078 ceph-mon[52919]: pgmap v246: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 640 op/s; 181/1626 objects degraded (11.132%) 2024-05-20T22:31:23.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:23 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:31:23.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:23 smithi078 ceph-mon[52919]: osd.5 [v2:172.21.15.78:6808/2845769459,v1:172.21.15.78:6809/2845769459] boot 2024-05-20T22:31:23.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:23 smithi078 ceph-mon[52919]: osdmap e134: 8 total, 8 up, 8 in 2024-05-20T22:31:23.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:31:23.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:23 smithi006 ceph-mon[65753]: pgmap v246: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.1 MiB/s wr, 640 op/s; 181/1626 objects degraded (11.132%) 2024-05-20T22:31:23.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:23 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:31:23.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:23 smithi006 ceph-mon[65753]: osd.5 [v2:172.21.15.78:6808/2845769459,v1:172.21.15.78:6809/2845769459] boot 2024-05-20T22:31:23.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:23 smithi006 ceph-mon[65753]: osdmap e134: 8 total, 8 up, 8 in 2024-05-20T22:31:23.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-05-20T22:31:24.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260307 10.11 MB/sec execute 510 sec latency 1432.860 ms 2024-05-20T22:31:24.695 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:31:24.942 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:24 smithi006 ceph-mon[65753]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 620 op/s; 184/1662 objects degraded (11.071%) 2024-05-20T22:31:24.942 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:24 smithi006 ceph-mon[65753]: osdmap e135: 8 total, 8 up, 8 in 2024-05-20T22:31:25.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260364 10.09 MB/sec execute 511 sec latency 2145.998 ms 2024-05-20T22:31:25.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:24 smithi078 ceph-mon[52919]: pgmap v249: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 620 op/s; 184/1662 objects degraded (11.071%) 2024-05-20T22:31:25.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:24 smithi078 ceph-mon[52919]: osdmap e135: 8 total, 8 up, 8 in 2024-05-20T22:31:25.671 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:31:25.671 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (8m) 88s ago 17m 14.7M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:31:25.671 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (4m) 88s ago 17m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:31:25.671 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (4m) 10s ago 16m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:31:25.671 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (7m) 88s ago 16m 43.8M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:31:25.671 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (11m) 88s ago 11m 37.6M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (11m) 10s ago 11m 14.0M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (7m) 88s ago 18m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (6m) 10s ago 16m 453M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (6m) 88s ago 18m 52.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (5m) 10s ago 16m 45.0M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (10m) 88s ago 10m 283M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (10m) 10s ago 10m 88.0M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (7m) 88s ago 16m 11.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (7m) 10s ago 15m 19.7M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (4m) 88s ago 15m 358M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (3m) 88s ago 15m 287M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (2m) 88s ago 14m 196M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (99s) 88s ago 14m 11.8M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:31:25.672 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (53s) 10s ago 13m 273M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:31:25.673 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (15s) 10s ago 13m 15.1M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:31:25.673 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (12m) 10s ago 12m 531M 2704M 17.2.0 e1d6a67b021e f15584afade4 2024-05-20T22:31:25.673 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (12m) 10s ago 12m 606M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:31:25.673 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (6m) 88s ago 15m 68.0M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:31:25.699 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:25 smithi006 ceph-mon[65753]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:25.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:25 smithi006 ceph-mon[65753]: osdmap e136: 8 total, 8 up, 8 in 2024-05-20T22:31:25.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:25 smithi006 ceph-mon[65753]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:25.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:25 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 184/1662 objects degraded (11.071%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:25.700 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:25 smithi006 ceph-mon[65753]: osdmap e137: 8 total, 8 up, 8 in 2024-05-20T22:31:26.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260628 10.09 MB/sec execute 512 sec latency 436.857 ms 2024-05-20T22:31:26.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:25 smithi078 ceph-mon[52919]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:26.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:25 smithi078 ceph-mon[52919]: osdmap e136: 8 total, 8 up, 8 in 2024-05-20T22:31:26.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:25 smithi078 ceph-mon[52919]: from='client.44327 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:26.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:25 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 184/1662 objects degraded (11.071%), 23 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:26.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:25 smithi078 ceph-mon[52919]: osdmap e137: 8 total, 8 up, 8 in 2024-05-20T22:31:26.184 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:31:26.184 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:31:26.184 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:31:26.184 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 6 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 10 2024-05-20T22:31:26.185 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:31:26.186 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:31:26.593 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "12/23 daemons upgraded", 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:31:26.594 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:31:26.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:26 smithi006 ceph-mon[65753]: pgmap v252: 97 pgs: 5 remapped+peering, 6 active+undersized, 16 active+undersized+degraded, 70 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.0 MiB/s wr, 218 op/s; 138/1665 objects degraded (8.288%); 22 B/s, 0 objects/s recovering 2024-05-20T22:31:26.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:26 smithi006 ceph-mon[65753]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:26.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:26 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3789355998' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:26.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:26 smithi006 ceph-mon[65753]: osdmap e138: 8 total, 8 up, 8 in 2024-05-20T22:31:27.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 260853 10.08 MB/sec execute 513 sec latency 997.487 ms 2024-05-20T22:31:27.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:26 smithi078 ceph-mon[52919]: pgmap v252: 97 pgs: 5 remapped+peering, 6 active+undersized, 16 active+undersized+degraded, 70 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 3.0 MiB/s wr, 218 op/s; 138/1665 objects degraded (8.288%); 22 B/s, 0 objects/s recovering 2024-05-20T22:31:27.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:26 smithi078 ceph-mon[52919]: from='client.44331 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:27.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:26 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3789355998' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:27.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:26 smithi078 ceph-mon[52919]: osdmap e138: 8 total, 8 up, 8 in 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN Degraded data redundancy: 138/1665 objects degraded (8.288%), 16 pgs degraded 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 138/1665 objects degraded (8.288%), 16 pgs degraded 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.12 is active+undersized+degraded, acting [3,2] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-05-20T22:31:27.180 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-05-20T22:31:27.181 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-05-20T22:31:27.181 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-05-20T22:31:27.181 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-05-20T22:31:27.181 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-05-20T22:31:27.181 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-05-20T22:31:27.181 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-05-20T22:31:27.740 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:27 smithi006 ceph-mon[65753]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:27.741 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:27 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2263914048' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:31:28.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261248 10.08 MB/sec execute 514 sec latency 55.314 ms 2024-05-20T22:31:28.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:27 smithi078 ceph-mon[52919]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:28.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:27 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2263914048' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:31:28.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:28 smithi006 ceph-mon[65753]: pgmap v255: 97 pgs: 3 peering, 1 remapped+peering, 93 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 472 op/s; 4.4 MiB/s, 0 keys/s, 57 objects/s recovering 2024-05-20T22:31:28.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:28 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 138/1665 objects degraded (8.288%), 16 pgs degraded) 2024-05-20T22:31:28.771 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:28 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:31:29.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261571 10.08 MB/sec execute 515 sec latency 45.606 ms 2024-05-20T22:31:29.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:28 smithi078 ceph-mon[52919]: pgmap v255: 97 pgs: 3 peering, 1 remapped+peering, 93 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.4 MiB/s wr, 472 op/s; 4.4 MiB/s, 0 keys/s, 57 objects/s recovering 2024-05-20T22:31:29.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:28 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 138/1665 objects degraded (8.288%), 16 pgs degraded) 2024-05-20T22:31:29.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:28 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:31:30.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 261970 10.08 MB/sec execute 516 sec latency 28.640 ms 2024-05-20T22:31:30.789 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:30 smithi006 ceph-mon[65753]: pgmap v256: 97 pgs: 3 peering, 94 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.1 MiB/s wr, 327 op/s; 3.2 MiB/s, 0 keys/s, 45 objects/s recovering 2024-05-20T22:31:30.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:30.790 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:31.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 262362 10.08 MB/sec execute 517 sec latency 40.039 ms 2024-05-20T22:31:31.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:30 smithi078 ceph-mon[52919]: pgmap v256: 97 pgs: 3 peering, 94 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.1 MiB/s wr, 327 op/s; 3.2 MiB/s, 0 keys/s, 45 objects/s recovering 2024-05-20T22:31:31.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:31.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:32.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 262692 10.08 MB/sec execute 518 sec latency 37.223 ms 2024-05-20T22:31:32.815 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:32 smithi006 ceph-mon[65753]: pgmap v257: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.4 MiB/s wr, 259 op/s; 2.7 MiB/s, 0 keys/s, 38 objects/s recovering 2024-05-20T22:31:33.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 263046 10.09 MB/sec execute 519 sec latency 42.082 ms 2024-05-20T22:31:33.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:32 smithi078 ceph-mon[52919]: pgmap v257: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.4 MiB/s wr, 259 op/s; 2.7 MiB/s, 0 keys/s, 38 objects/s recovering 2024-05-20T22:31:34.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 263373 10.09 MB/sec execute 520 sec latency 43.441 ms 2024-05-20T22:31:34.845 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:34 smithi006 ceph-mon[65753]: pgmap v258: 97 pgs: 97 active+clean; 354 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 506 op/s; 2.4 MiB/s, 0 keys/s, 33 objects/s recovering 2024-05-20T22:31:35.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 263652 10.07 MB/sec execute 521 sec latency 51.269 ms 2024-05-20T22:31:35.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:34 smithi078 ceph-mon[52919]: pgmap v258: 97 pgs: 97 active+clean; 354 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 506 op/s; 2.4 MiB/s, 0 keys/s, 33 objects/s recovering 2024-05-20T22:31:36.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 263936 10.06 MB/sec execute 522 sec latency 66.907 ms 2024-05-20T22:31:36.814 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:36 smithi078 ceph-mon[52919]: pgmap v259: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.5 MiB/s wr, 479 op/s; 1.9 MiB/s, 0 keys/s, 26 objects/s recovering 2024-05-20T22:31:36.814 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:36 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:36.814 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:36 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:36.814 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:36 smithi078 ceph-mon[52919]: Upgrade: osd.6 is safe to restart 2024-05-20T22:31:36.903 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:36 smithi006 ceph-mon[65753]: pgmap v259: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.5 MiB/s wr, 479 op/s; 1.9 MiB/s, 0 keys/s, 26 objects/s recovering 2024-05-20T22:31:36.904 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:36 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:36.904 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:36 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-05-20T22:31:36.904 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:36 smithi006 ceph-mon[65753]: Upgrade: osd.6 is safe to restart 2024-05-20T22:31:37.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 264236 10.06 MB/sec execute 523 sec latency 53.337 ms 2024-05-20T22:31:38.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 264611 10.06 MB/sec execute 524 sec latency 55.081 ms 2024-05-20T22:31:39.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 264940 10.06 MB/sec execute 525 sec latency 63.325 ms 2024-05-20T22:31:39.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:38 smithi006 ceph-mon[65753]: pgmap v260: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 461 op/s; 968 KiB/s, 15 objects/s recovering 2024-05-20T22:31:39.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:38 smithi006 ceph-mon[65753]: Upgrade: Updating osd.6 2024-05-20T22:31:39.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:39.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:31:39.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:39.218 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:38 smithi006 ceph-mon[65753]: Deploying daemon osd.6 on smithi078 2024-05-20T22:31:39.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:38 smithi078 ceph-mon[52919]: pgmap v260: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 461 op/s; 968 KiB/s, 15 objects/s recovering 2024-05-20T22:31:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:38 smithi078 ceph-mon[52919]: Upgrade: Updating osd.6 2024-05-20T22:31:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-05-20T22:31:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:31:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:38 smithi078 ceph-mon[52919]: Deploying daemon osd.6 on smithi078 2024-05-20T22:31:40.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 265263 10.06 MB/sec execute 526 sec latency 41.500 ms 2024-05-20T22:31:41.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:40 smithi006 ceph-mon[65753]: pgmap v261: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 108 KiB/s, 3 objects/s recovering 2024-05-20T22:31:41.001 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:40 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:41.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 265544 10.05 MB/sec execute 527 sec latency 61.090 ms 2024-05-20T22:31:41.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:40 smithi078 ceph-mon[52919]: pgmap v261: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 108 KiB/s, 3 objects/s recovering 2024-05-20T22:31:41.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:40 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:42.077 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:41 smithi078 systemd[1]: Stopping Ceph osd.6 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:31:42.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 265828 10.04 MB/sec execute 528 sec latency 43.787 ms 2024-05-20T22:31:42.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:42 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[37335]: 2024-05-20T22:31:42.205+0000 7feb8edeb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:31:42.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:42 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[37335]: 2024-05-20T22:31:42.205+0000 7feb8edeb700 -1 osd.6 138 *** Got signal Terminated *** 2024-05-20T22:31:42.396 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:42 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[37335]: 2024-05-20T22:31:42.205+0000 7feb8edeb700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:31:43.045 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:42 smithi006 ceph-mon[65753]: pgmap v262: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-05-20T22:31:43.045 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:42 smithi006 ceph-mon[65753]: osd.6 marked itself down and dead 2024-05-20T22:31:43.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 266122 10.04 MB/sec execute 529 sec latency 101.614 ms 2024-05-20T22:31:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:42 smithi078 ceph-mon[52919]: pgmap v262: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-05-20T22:31:43.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:42 smithi078 ceph-mon[52919]: osd.6 marked itself down and dead 2024-05-20T22:31:43.895 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:43 smithi078 podman[61851]: 2024-05-20 22:31:43.629139962 +0000 UTC m=+1.657914475 container died f15584afade40f7ebc095fb515c11ee6f3d1785f4082f3c2c3739d2f69fd4461 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, version=8, GIT_BRANCH=HEAD, vcs-type=git, release=754, maintainer=Guillaume Abrioux , ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, name=centos-stream, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-05-20T22:31:44.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 266154 10.03 MB/sec execute 530 sec latency 1021.138 ms 2024-05-20T22:31:44.228 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:43 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:31:44.229 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:43 smithi006 ceph-mon[65753]: osdmap e139: 8 total, 7 up, 8 in 2024-05-20T22:31:44.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:43 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:31:44.242 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:43 smithi078 ceph-mon[52919]: osdmap e139: 8 total, 7 up, 8 in 2024-05-20T22:31:44.510 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:44 smithi078 podman[61851]: 2024-05-20 22:31:44.24740597 +0000 UTC m=+2.276180485 container cleanup f15584afade40f7ebc095fb515c11ee6f3d1785f4082f3c2c3739d2f69fd4461 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git) 2024-05-20T22:31:44.511 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:44 smithi078 bash[61851]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6 2024-05-20T22:31:44.877 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:44 smithi078 podman[61879]: 2024-05-20 22:31:44.515752402 +0000 UTC m=+0.878212143 container remove f15584afade40f7ebc095fb515c11ee6f3d1785f4082f3c2c3739d2f69fd4461 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_CLEAN=True, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-05-20T22:31:44.988 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:44 smithi006 ceph-mon[65753]: pgmap v264: 97 pgs: 10 stale+active+clean, 87 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.2 MiB/s wr, 388 op/s 2024-05-20T22:31:44.988 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:44 smithi006 ceph-mon[65753]: osdmap e140: 8 total, 7 up, 8 in 2024-05-20T22:31:45.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 266506 10.03 MB/sec execute 531 sec latency 54.769 ms 2024-05-20T22:31:45.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:44 smithi078 ceph-mon[52919]: pgmap v264: 97 pgs: 10 stale+active+clean, 87 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.2 MiB/s wr, 388 op/s 2024-05-20T22:31:45.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:44 smithi078 ceph-mon[52919]: osdmap e140: 8 total, 7 up, 8 in 2024-05-20T22:31:45.146 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:44 smithi078 podman[61966]: 2024-05-20 22:31:44.785509461 +0000 UTC m=+0.022007957 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:31:45.146 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:44 smithi078 podman[61966]: 2024-05-20 22:31:44.900255604 +0000 UTC m=+0.136754154 container create 675efee46cdb498f6d4d36c91d0935a832e6b315baf5e9bb26e7e8d5ddd7b56b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, io.buildah.version=1.33.5) 2024-05-20T22:31:45.565 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:45 smithi078 podman[61966]: 2024-05-20 22:31:45.204097016 +0000 UTC m=+0.440595503 container init 675efee46cdb498f6d4d36c91d0935a832e6b315baf5e9bb26e7e8d5ddd7b56b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-05-20T22:31:45.565 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:45 smithi078 podman[61966]: 2024-05-20 22:31:45.209250935 +0000 UTC m=+0.445749422 container start 675efee46cdb498f6d4d36c91d0935a832e6b315baf5e9bb26e7e8d5ddd7b56b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:31:45.565 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:45 smithi078 podman[61966]: 2024-05-20 22:31:45.312070734 +0000 UTC m=+0.548569225 container attach 675efee46cdb498f6d4d36c91d0935a832e6b315baf5e9bb26e7e8d5ddd7b56b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-05-20T22:31:45.895 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:45 smithi078 podman[61966]: 2024-05-20 22:31:45.57034588 +0000 UTC m=+0.806844364 container died 675efee46cdb498f6d4d36c91d0935a832e6b315baf5e9bb26e7e8d5ddd7b56b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5) 2024-05-20T22:31:46.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 266810 10.03 MB/sec execute 532 sec latency 59.411 ms 2024-05-20T22:31:46.542 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 podman[61992]: 2024-05-20 22:31:46.20848295 +0000 UTC m=+0.629654615 container remove 675efee46cdb498f6d4d36c91d0935a832e6b315baf5e9bb26e7e8d5ddd7b56b (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:31:46.542 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.6.service: Deactivated successfully. 2024-05-20T22:31:46.542 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 systemd[1]: Stopped Ceph osd.6 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:31:46.542 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.6.service: Consumed 4min 42.700s CPU time. 2024-05-20T22:31:46.542 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 systemd[1]: Starting Ceph osd.6 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:31:46.676 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:46.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:46.677 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:46 smithi006 ceph-mon[65753]: pgmap v266: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 9 stale+active+clean, 84 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 403 op/s; 25/1629 objects degraded (1.535%) 2024-05-20T22:31:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:31:46.851 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:46 smithi078 ceph-mon[52919]: pgmap v266: 97 pgs: 1 active+undersized, 3 active+undersized+degraded, 9 stale+active+clean, 84 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 403 op/s; 25/1629 objects degraded (1.535%) 2024-05-20T22:31:46.851 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 podman[62088]: 2024-05-20 22:31:46.591245755 +0000 UTC m=+0.019043664 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:31:46.851 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 podman[62088]: 2024-05-20 22:31:46.726970208 +0000 UTC m=+0.154768172 container create f65e8d62d2a4d6a0d5e8d95ff7a27f67178e16e0044428c9f6302612c31017a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-20T22:31:47.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 267115 10.02 MB/sec execute 533 sec latency 90.213 ms 2024-05-20T22:31:47.145 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:46 smithi078 podman[62088]: 2024-05-20 22:31:46.996592651 +0000 UTC m=+0.424390548 container init f65e8d62d2a4d6a0d5e8d95ff7a27f67178e16e0044428c9f6302612c31017a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-05-20T22:31:47.145 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 podman[62088]: 2024-05-20 22:31:47.000590666 +0000 UTC m=+0.428388581 container start f65e8d62d2a4d6a0d5e8d95ff7a27f67178e16e0044428c9f6302612c31017a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:31:47.145 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 podman[62088]: 2024-05-20 22:31:47.096609237 +0000 UTC m=+0.524407140 container attach f65e8d62d2a4d6a0d5e8d95ff7a27f67178e16e0044428c9f6302612c31017a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514) 2024-05-20T22:31:47.789 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:47 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 25/1629 objects degraded (1.535%), 3 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:47.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:47 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 25/1629 objects degraded (1.535%), 3 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate[62103]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate[62103]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 bash[62088]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 bash[62088]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 bash[62088]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 bash[62088]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 bash[62088]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 bash[62088]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 bash[62088]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate[62103]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate[62103]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate[62103]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-05-20T22:31:47.896 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate[62103]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-05-20T22:31:47.897 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate[62103]: --> ceph-volume raw activate successful for osd ID: 6 2024-05-20T22:31:47.897 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:47 smithi078 podman[62088]: 2024-05-20 22:31:47.557375381 +0000 UTC m=+0.985173279 container died f65e8d62d2a4d6a0d5e8d95ff7a27f67178e16e0044428c9f6302612c31017a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-20T22:31:48.104 INFO:teuthology.orchestra.run.smithi006.stdout: 5 267434 10.01 MB/sec execute 534 sec latency 42.614 ms 2024-05-20T22:31:48.574 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:48 smithi006 ceph-mon[65753]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.2 MiB/s wr, 521 op/s; 190/1623 objects degraded (11.707%) 2024-05-20T22:31:48.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:48 smithi078 ceph-mon[52919]: pgmap v267: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.2 MiB/s wr, 521 op/s; 190/1623 objects degraded (11.707%) 2024-05-20T22:31:48.850 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:48 smithi078 podman[62223]: 2024-05-20 22:31:48.627791495 +0000 UTC m=+1.064416647 container remove f65e8d62d2a4d6a0d5e8d95ff7a27f67178e16e0044428c9f6302612c31017a0 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-05-20T22:31:49.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 267784 10.01 MB/sec execute 535 sec latency 40.747 ms 2024-05-20T22:31:49.145 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:48 smithi078 podman[62282]: 2024-05-20 22:31:48.757912097 +0000 UTC m=+0.023513803 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:31:49.145 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:48 smithi078 podman[62282]: 2024-05-20 22:31:48.903520139 +0000 UTC m=+0.169121822 container create 811c56af46dd0a057b56eff66129caa1557c0b5d429774a0f3c86c8a38333224 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True) 2024-05-20T22:31:49.646 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:49 smithi078 podman[62282]: 2024-05-20 22:31:49.164994273 +0000 UTC m=+0.430595965 container init 811c56af46dd0a057b56eff66129caa1557c0b5d429774a0f3c86c8a38333224 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, io.buildah.version=1.33.5) 2024-05-20T22:31:49.646 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:49 smithi078 podman[62282]: 2024-05-20 22:31:49.168784775 +0000 UTC m=+0.434386467 container start 811c56af46dd0a057b56eff66129caa1557c0b5d429774a0f3c86c8a38333224 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514) 2024-05-20T22:31:49.646 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:49 smithi078 ceph-osd[62300]: -- 172.21.15.78:0/1162260971 <== mon.1 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55559d671680 con 0x55559d63b000 2024-05-20T22:31:49.646 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:49 smithi078 bash[62282]: 811c56af46dd0a057b56eff66129caa1557c0b5d429774a0f3c86c8a38333224 2024-05-20T22:31:49.646 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:49 smithi078 systemd[1]: Started Ceph osd.6 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:31:50.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 268142 10.01 MB/sec execute 536 sec latency 55.313 ms 2024-05-20T22:31:50.145 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:49 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:31:49.717+0000 7ff3500947c0 -1 Falling back to public interface 2024-05-20T22:31:50.378 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:50.378 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:50.378 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:50 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:50.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:50.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:50.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:50 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:31:51.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 268473 10.01 MB/sec execute 537 sec latency 82.454 ms 2024-05-20T22:31:51.374 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:51 smithi078 ceph-mon[52919]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 472 op/s; 187/1623 objects degraded (11.522%) 2024-05-20T22:31:51.420 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:51 smithi006 ceph-mon[65753]: pgmap v268: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 472 op/s; 187/1623 objects degraded (11.522%) 2024-05-20T22:31:52.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 268825 10.01 MB/sec execute 538 sec latency 54.851 ms 2024-05-20T22:31:52.439 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:52 smithi006 ceph-mon[65753]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 4.4 MiB/s wr, 466 op/s; 190/1659 objects degraded (11.453%) 2024-05-20T22:31:52.440 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:52 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 190/1659 objects degraded (11.453%), 25 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:52.540 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:52 smithi078 ceph-mon[52919]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 4.4 MiB/s wr, 466 op/s; 190/1659 objects degraded (11.453%) 2024-05-20T22:31:52.540 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:52 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 190/1659 objects degraded (11.453%), 25 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:53.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 269212 10.01 MB/sec execute 539 sec latency 47.616 ms 2024-05-20T22:31:54.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 269574 10.01 MB/sec execute 540 sec latency 45.451 ms 2024-05-20T22:31:55.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 269979 10.02 MB/sec execute 541 sec latency 60.770 ms 2024-05-20T22:31:55.382 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:55 smithi006 ceph-mon[65753]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 489 op/s; 186/1623 objects degraded (11.460%) 2024-05-20T22:31:55.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:55.383 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:55.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:55 smithi078 ceph-mon[52919]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 489 op/s; 186/1623 objects degraded (11.460%) 2024-05-20T22:31:55.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:55.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:56.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 270349 10.02 MB/sec execute 542 sec latency 55.695 ms 2024-05-20T22:31:56.682 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:56 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:56.683 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:56 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:56.683 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:56 smithi006 ceph-mon[65753]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 463 op/s; 188/1635 objects degraded (11.498%) 2024-05-20T22:31:56.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:56 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:56.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:56 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:31:56.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:56 smithi078 ceph-mon[52919]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 463 op/s; 188/1635 objects degraded (11.498%) 2024-05-20T22:31:57.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 270748 10.02 MB/sec execute 543 sec latency 67.296 ms 2024-05-20T22:31:57.705 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:57 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 190/1656 objects degraded (11.473%), 25 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:57.710 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:31:57.862 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:57 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 190/1656 objects degraded (11.473%), 25 pgs degraded (PG_DEGRADED) 2024-05-20T22:31:58.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 271072 10.01 MB/sec execute 544 sec latency 41.288 ms 2024-05-20T22:31:58.145 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:57 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:31:57.874+0000 7ff3500947c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-05-20T22:31:58.698 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:31:58.698 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (8m) 2m ago 17m 14.7M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:31:58.698 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (5m) 2m ago 17m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:31:58.698 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (5m) 4s ago 16m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:31:58.698 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (8m) 2m ago 17m 43.8M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:31:58.698 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (11m) 2m ago 11m 37.6M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (11m) 4s ago 11m 14.1M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (8m) 2m ago 18m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (7m) 4s ago 16m 453M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (6m) 2m ago 19m 52.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (6m) 4s ago 16m 44.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (11m) 2m ago 11m 283M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (11m) 4s ago 11m 88.0M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (8m) 2m ago 17m 11.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 4s ago 16m 16.9M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (4m) 2m ago 16m 358M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (3m) 2m ago 15m 287M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (3m) 2m ago 15m 196M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (2m) 2m ago 14m 11.8M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:31:58.699 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (86s) 4s ago 14m 296M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:31:58.700 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (48s) 4s ago 13m 273M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:31:58.700 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (9s) 4s ago 13m 12.3M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:31:58.700 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (13m) 4s ago 13m 660M 2704M 17.2.0 e1d6a67b021e ca10a5edd069 2024-05-20T22:31:58.700 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 2m ago 16m 68.0M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:31:58.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:58 smithi006 ceph-mon[65753]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.7 MiB/s wr, 579 op/s; 190/1656 objects degraded (11.473%) 2024-05-20T22:31:58.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:58 smithi006 ceph-mon[65753]: from='client.44349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:58.723 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:58 smithi006 ceph-mon[65753]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:59.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 271463 10.01 MB/sec execute 545 sec latency 49.901 ms 2024-05-20T22:31:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:58 smithi078 ceph-mon[52919]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.7 MiB/s wr, 579 op/s; 190/1656 objects degraded (11.473%) 2024-05-20T22:31:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:58 smithi078 ceph-mon[52919]: from='client.44349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:58 smithi078 ceph-mon[52919]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:31:59.359 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 6 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 10 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:31:59.360 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:31:59.697 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:31:59 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:31:59.446+0000 7ff3500947c0 -1 osd.6 138 log_to_monitors true 2024-05-20T22:31:59.748 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:59 smithi006 ceph-mon[65753]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:31:59.748 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:59 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3247175875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:31:59.748 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:59 smithi006 ceph-mon[65753]: from='osd.6 [v2:172.21.15.78:6816/1956602318,v1:172.21.15.78:6817/1956602318]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:31:59.748 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:31:59 smithi006 ceph-mon[65753]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:31:59.831 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:31:59.831 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:31:59.831 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:31:59.831 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr" 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "13/23 daemons upgraded", 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:31:59.832 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:32:00.105 INFO:teuthology.orchestra.run.smithi006.stdout: 5 271762 10.01 MB/sec execute 546 sec latency 139.659 ms 2024-05-20T22:32:00.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:59 smithi078 ceph-mon[52919]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:00.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:59 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3247175875' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:00.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:59 smithi078 ceph-mon[52919]: from='osd.6 [v2:172.21.15.78:6816/1956602318,v1:172.21.15.78:6817/1956602318]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:32:00.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:31:59 smithi078 ceph-mon[52919]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-05-20T22:32:00.375 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 191/1668 objects degraded (11.451%), 25 pgs degraded 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: osd.6 (root=default,host=smithi078) is down 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 191/1668 objects degraded (11.451%), 25 pgs degraded 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-05-20T22:32:00.376 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-05-20T22:32:00.377 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-05-20T22:32:00.378 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-05-20T22:32:00.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:32:00 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:32:00.217+0000 7ff33d4e3700 -1 osd.6 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 467 op/s; 191/1668 objects degraded (11.451%) 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: osdmap e141: 8 total, 7 up, 8 in 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='osd.6 [v2:172.21.15.78:6816/1956602318,v1:172.21.15.78:6817/1956602318]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:00.760 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='osd.6 ' entity='osd.6' 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1129346257' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:00.761 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:01.112 INFO:teuthology.orchestra.run.smithi006.stdout: 5 272085 10.01 MB/sec execute 547 sec latency 75.675 ms 2024-05-20T22:32:01.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 467 op/s; 191/1668 objects degraded (11.451%) 2024-05-20T22:32:01.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: osdmap e141: 8 total, 7 up, 8 in 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='osd.6 [v2:172.21.15.78:6816/1956602318,v1:172.21.15.78:6817/1956602318]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='osd.6 ' entity='osd.6' 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:01.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1129346257' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:01.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:02.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:01 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:32:02.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:01 smithi006 ceph-mon[65753]: osd.6 [v2:172.21.15.78:6816/1956602318,v1:172.21.15.78:6817/1956602318] boot 2024-05-20T22:32:02.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:01 smithi006 ceph-mon[65753]: osdmap e142: 8 total, 8 up, 8 in 2024-05-20T22:32:02.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:32:02.113 INFO:teuthology.orchestra.run.smithi006.stdout: 5 272124 9.99 MB/sec execute 548 sec latency 909.940 ms 2024-05-20T22:32:02.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:01 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:32:02.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:01 smithi078 ceph-mon[52919]: osd.6 [v2:172.21.15.78:6816/1956602318,v1:172.21.15.78:6817/1956602318] boot 2024-05-20T22:32:02.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:01 smithi078 ceph-mon[52919]: osdmap e142: 8 total, 8 up, 8 in 2024-05-20T22:32:02.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-05-20T22:32:03.113 INFO:teuthology.orchestra.run.smithi006.stdout: 5 272124 9.97 MB/sec execute 549 sec latency 1909.982 ms 2024-05-20T22:32:03.467 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:03 smithi006 ceph-mon[65753]: pgmap v276: 97 pgs: 4 peering, 10 active+undersized, 21 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.0 MiB/s wr, 461 op/s; 154/1596 objects degraded (9.649%) 2024-05-20T22:32:03.467 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:03 smithi006 ceph-mon[65753]: osdmap e143: 8 total, 8 up, 8 in 2024-05-20T22:32:03.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:03 smithi078 ceph-mon[52919]: pgmap v276: 97 pgs: 4 peering, 10 active+undersized, 21 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 5.0 MiB/s wr, 461 op/s; 154/1596 objects degraded (9.649%) 2024-05-20T22:32:03.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:03 smithi078 ceph-mon[52919]: osdmap e143: 8 total, 8 up, 8 in 2024-05-20T22:32:04.113 INFO:teuthology.orchestra.run.smithi006.stdout: 5 272352 9.97 MB/sec execute 550 sec latency 2237.537 ms 2024-05-20T22:32:04.222 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:04 smithi006 ceph-mon[65753]: osdmap e144: 8 total, 8 up, 8 in 2024-05-20T22:32:04.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:04 smithi078 ceph-mon[52919]: osdmap e144: 8 total, 8 up, 8 in 2024-05-20T22:32:05.113 INFO:teuthology.orchestra.run.smithi006.stdout: 5 272394 9.95 MB/sec execute 551 sec latency 905.433 ms 2024-05-20T22:32:05.474 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:05 smithi006 ceph-mon[65753]: pgmap v279: 97 pgs: 1 active, 1 activating+degraded, 5 activating, 1 unknown, 14 peering, 2 active+undersized, 11 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 5.1 MiB/s wr, 506 op/s; 100/1590 objects degraded (6.289%) 2024-05-20T22:32:05.475 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:05 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 100/1590 objects degraded (6.289%), 12 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:05.475 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:05 smithi006 ceph-mon[65753]: osdmap e145: 8 total, 8 up, 8 in 2024-05-20T22:32:05.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:05 smithi078 ceph-mon[52919]: pgmap v279: 97 pgs: 1 active, 1 activating+degraded, 5 activating, 1 unknown, 14 peering, 2 active+undersized, 11 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 5.1 MiB/s wr, 506 op/s; 100/1590 objects degraded (6.289%) 2024-05-20T22:32:05.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:05 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 100/1590 objects degraded (6.289%), 12 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:05.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:05 smithi078 ceph-mon[52919]: osdmap e145: 8 total, 8 up, 8 in 2024-05-20T22:32:06.113 INFO:teuthology.orchestra.run.smithi006.stdout: 5 272684 9.95 MB/sec execute 552 sec latency 1064.962 ms 2024-05-20T22:32:06.482 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:06 smithi006 ceph-mon[65753]: osdmap e146: 8 total, 8 up, 8 in 2024-05-20T22:32:06.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:06 smithi078 ceph-mon[52919]: osdmap e146: 8 total, 8 up, 8 in 2024-05-20T22:32:07.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 273075 9.95 MB/sec execute 553 sec latency 33.579 ms 2024-05-20T22:32:07.243 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:07 smithi006 ceph-mon[65753]: pgmap v282: 97 pgs: 1 active, 1 activating+degraded, 5 activating, 1 unknown, 16 peering, 1 active+undersized+remapped, 72 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.6 MiB/s wr, 442 op/s; 30/1578 objects degraded (1.901%); 105 KiB/s, 65 keys/s, 11 objects/s recovering 2024-05-20T22:32:07.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:07 smithi078 ceph-mon[52919]: pgmap v282: 97 pgs: 1 active, 1 activating+degraded, 5 activating, 1 unknown, 16 peering, 1 active+undersized+remapped, 72 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.6 MiB/s wr, 442 op/s; 30/1578 objects degraded (1.901%); 105 KiB/s, 65 keys/s, 11 objects/s recovering 2024-05-20T22:32:08.116 INFO:teuthology.orchestra.run.smithi006.stdout: 5 273412 9.95 MB/sec execute 554 sec latency 53.444 ms 2024-05-20T22:32:08.503 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:08 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1578 objects degraded (1.901%), 1 pg degraded) 2024-05-20T22:32:08.503 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:08 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:32:08.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:08 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1578 objects degraded (1.901%), 1 pg degraded) 2024-05-20T22:32:08.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:08 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:32:09.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 273741 9.95 MB/sec execute 555 sec latency 49.821 ms 2024-05-20T22:32:09.516 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:09 smithi006 ceph-mon[65753]: pgmap v283: 97 pgs: 2 peering, 1 active+undersized+remapped, 94 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.9 MiB/s wr, 390 op/s; 1.7 MiB/s, 48 keys/s, 39 objects/s recovering 2024-05-20T22:32:09.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:09 smithi078 ceph-mon[52919]: pgmap v283: 97 pgs: 2 peering, 1 active+undersized+remapped, 94 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.9 MiB/s wr, 390 op/s; 1.7 MiB/s, 48 keys/s, 39 objects/s recovering 2024-05-20T22:32:10.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 274079 9.95 MB/sec execute 556 sec latency 48.082 ms 2024-05-20T22:32:11.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 274365 9.94 MB/sec execute 557 sec latency 49.082 ms 2024-05-20T22:32:11.311 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:11 smithi006 ceph-mon[65753]: pgmap v284: 97 pgs: 2 peering, 95 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.1 MiB/s wr, 206 op/s; 1.4 MiB/s, 40 keys/s, 32 objects/s recovering 2024-05-20T22:32:11.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:11 smithi078 ceph-mon[52919]: pgmap v284: 97 pgs: 2 peering, 95 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.1 MiB/s wr, 206 op/s; 1.4 MiB/s, 40 keys/s, 32 objects/s recovering 2024-05-20T22:32:12.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 274639 9.93 MB/sec execute 558 sec latency 40.870 ms 2024-05-20T22:32:13.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 274945 9.93 MB/sec execute 559 sec latency 47.126 ms 2024-05-20T22:32:13.352 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:13 smithi006 ceph-mon[65753]: pgmap v285: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.8 MiB/s wr, 217 op/s; 1.6 MiB/s, 32 keys/s, 30 objects/s recovering 2024-05-20T22:32:13.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:13 smithi078 ceph-mon[52919]: pgmap v285: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 2.8 MiB/s wr, 217 op/s; 1.6 MiB/s, 32 keys/s, 30 objects/s recovering 2024-05-20T22:32:14.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 275347 9.93 MB/sec execute 560 sec latency 40.162 ms 2024-05-20T22:32:15.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 275651 9.93 MB/sec execute 561 sec latency 41.366 ms 2024-05-20T22:32:15.424 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:15 smithi006 ceph-mon[65753]: pgmap v286: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 387 op/s; 1.4 MiB/s, 23 objects/s recovering 2024-05-20T22:32:15.424 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:15.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:15 smithi078 ceph-mon[52919]: pgmap v286: 97 pgs: 97 active+clean; 350 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 387 op/s; 1.4 MiB/s, 23 objects/s recovering 2024-05-20T22:32:15.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:16.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 276025 9.93 MB/sec execute 562 sec latency 46.213 ms 2024-05-20T22:32:16.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:16 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:16.436 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:16 smithi006 ceph-mon[65753]: Upgrade: osd.7 is safe to restart 2024-05-20T22:32:16.437 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:16.437 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:16.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:16 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-05-20T22:32:16.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:16 smithi078 ceph-mon[52919]: Upgrade: osd.7 is safe to restart 2024-05-20T22:32:16.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:16.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:17.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 276354 9.92 MB/sec execute 563 sec latency 42.385 ms 2024-05-20T22:32:17.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:17 smithi006 ceph-mon[65753]: pgmap v287: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 357 op/s; 1.2 MiB/s, 18 objects/s recovering 2024-05-20T22:32:17.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:17.452 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:32:17.453 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:17 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:17.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:17 smithi078 ceph-mon[52919]: pgmap v287: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.6 MiB/s wr, 357 op/s; 1.2 MiB/s, 18 objects/s recovering 2024-05-20T22:32:17.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:17.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-05-20T22:32:17.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:17 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:18.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 276625 9.92 MB/sec execute 564 sec latency 67.842 ms 2024-05-20T22:32:18.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:18 smithi078 ceph-mon[52919]: Upgrade: Updating osd.7 2024-05-20T22:32:18.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:18 smithi078 ceph-mon[52919]: Deploying daemon osd.7 on smithi078 2024-05-20T22:32:18.472 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:18 smithi006 ceph-mon[65753]: Upgrade: Updating osd.7 2024-05-20T22:32:18.472 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:18 smithi006 ceph-mon[65753]: Deploying daemon osd.7 on smithi078 2024-05-20T22:32:19.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 276878 9.92 MB/sec execute 565 sec latency 52.784 ms 2024-05-20T22:32:19.486 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:19 smithi006 ceph-mon[65753]: pgmap v288: 97 pgs: 97 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 486 op/s; 1.1 MiB/s, 16 objects/s recovering 2024-05-20T22:32:19.600 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:19 smithi078 ceph-mon[52919]: pgmap v288: 97 pgs: 97 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 486 op/s; 1.1 MiB/s, 16 objects/s recovering 2024-05-20T22:32:20.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 277198 9.92 MB/sec execute 566 sec latency 47.213 ms 2024-05-20T22:32:20.895 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:20 smithi078 systemd[1]: Stopping Ceph osd.7 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:32:21.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 277608 9.92 MB/sec execute 567 sec latency 44.055 ms 2024-05-20T22:32:21.316 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[40390]: 2024-05-20T22:32:20.973+0000 7f516392b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:32:21.316 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[40390]: 2024-05-20T22:32:20.973+0000 7f516392b700 -1 osd.7 146 *** Got signal Terminated *** 2024-05-20T22:32:21.316 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[40390]: 2024-05-20T22:32:20.973+0000 7f516392b700 -1 osd.7 146 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:32:21.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:21 smithi006 ceph-mon[65753]: pgmap v289: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 401 op/s; 361 KiB/s, 2 objects/s recovering 2024-05-20T22:32:21.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:21 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:21.517 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:21 smithi006 ceph-mon[65753]: osd.7 marked itself down and dead 2024-05-20T22:32:21.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:21 smithi078 ceph-mon[52919]: pgmap v289: 97 pgs: 97 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 401 op/s; 361 KiB/s, 2 objects/s recovering 2024-05-20T22:32:21.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:21 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:21.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:21 smithi078 ceph-mon[52919]: osd.7 marked itself down and dead 2024-05-20T22:32:22.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 277889 9.91 MB/sec execute 568 sec latency 475.157 ms 2024-05-20T22:32:22.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:22 smithi078 ceph-mon[52919]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:32:22.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:22 smithi078 ceph-mon[52919]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-20T22:32:22.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:22 smithi078 ceph-mon[52919]: osdmap e147: 8 total, 7 up, 8 in 2024-05-20T22:32:22.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:22 smithi078 podman[64113]: 2024-05-20 22:32:22.350666131 +0000 UTC m=+1.497162729 container died ca10a5edd0690e028ca832e566b70d92e51ff67f69e27e34fc35e3d194b608d3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, name=centos-stream, release=754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_BRANCH=HEAD, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870) 2024-05-20T22:32:22.684 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:22 smithi006 ceph-mon[65753]: Health check failed: 1 osds down (OSD_DOWN) 2024-05-20T22:32:22.684 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:22 smithi006 ceph-mon[65753]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-05-20T22:32:22.684 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:22 smithi006 ceph-mon[65753]: osdmap e147: 8 total, 7 up, 8 in 2024-05-20T22:32:23.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 278007 9.90 MB/sec execute 569 sec latency 1026.966 ms 2024-05-20T22:32:23.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:22 smithi078 podman[64113]: 2024-05-20 22:32:22.766135757 +0000 UTC m=+1.912632351 container cleanup ca10a5edd0690e028ca832e566b70d92e51ff67f69e27e34fc35e3d194b608d3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, architecture=x86_64, name=centos-stream, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, release=754, io.buildah.version=1.19.8) 2024-05-20T22:32:23.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:22 smithi078 bash[64113]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7 2024-05-20T22:32:23.446 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:23 smithi006 ceph-mon[65753]: pgmap v290: 97 pgs: 97 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 532 op/s; 361 KiB/s, 2 objects/s recovering 2024-05-20T22:32:23.446 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:23 smithi006 ceph-mon[65753]: osdmap e148: 8 total, 7 up, 8 in 2024-05-20T22:32:23.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:23 smithi078 ceph-mon[52919]: pgmap v290: 97 pgs: 97 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.1 MiB/s wr, 532 op/s; 361 KiB/s, 2 objects/s recovering 2024-05-20T22:32:23.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:23 smithi078 ceph-mon[52919]: osdmap e148: 8 total, 7 up, 8 in 2024-05-20T22:32:23.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:23 smithi078 podman[64128]: 2024-05-20 22:32:23.468548328 +0000 UTC m=+1.118972988 container remove ca10a5edd0690e028ca832e566b70d92e51ff67f69e27e34fc35e3d194b608d3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, io.k8s.display-name=CentOS Stream 8, release=754, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-05-20T22:32:24.093 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:23 smithi078 podman[64218]: 2024-05-20 22:32:23.716029384 +0000 UTC m=+0.018859936 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:32:24.093 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:23 smithi078 podman[64218]: 2024-05-20 22:32:23.844732543 +0000 UTC m=+0.147563085 container create 59bef2e63cefec116c43507f41766bc9169927c14b9b724af6df1774f02a4ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:32:24.093 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:24 smithi078 podman[64218]: 2024-05-20 22:32:24.098295045 +0000 UTC m=+0.401125599 container init 59bef2e63cefec116c43507f41766bc9169927c14b9b724af6df1774f02a4ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-20T22:32:24.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 278355 9.90 MB/sec execute 570 sec latency 54.764 ms 2024-05-20T22:32:24.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:24 smithi078 podman[64218]: 2024-05-20 22:32:24.102634235 +0000 UTC m=+0.405464775 container start 59bef2e63cefec116c43507f41766bc9169927c14b9b724af6df1774f02a4ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux ) 2024-05-20T22:32:24.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:24 smithi078 podman[64218]: 2024-05-20 22:32:24.22344996 +0000 UTC m=+0.526280500 container attach 59bef2e63cefec116c43507f41766bc9169927c14b9b724af6df1774f02a4ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-05-20T22:32:24.895 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:24 smithi078 podman[64218]: 2024-05-20 22:32:24.445233098 +0000 UTC m=+0.748063648 container died 59bef2e63cefec116c43507f41766bc9169927c14b9b724af6df1774f02a4ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=) 2024-05-20T22:32:25.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 278834 9.90 MB/sec execute 571 sec latency 63.108 ms 2024-05-20T22:32:25.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:25 smithi078 ceph-mon[52919]: pgmap v293: 97 pgs: 23 peering, 5 stale+active+clean, 69 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-05-20T22:32:25.503 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:25 smithi006 ceph-mon[65753]: pgmap v293: 97 pgs: 23 peering, 5 stale+active+clean, 69 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-05-20T22:32:25.691 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:25 smithi078 podman[64243]: 2024-05-20 22:32:25.445818041 +0000 UTC m=+0.993821046 container remove 59bef2e63cefec116c43507f41766bc9169927c14b9b724af6df1774f02a4ea8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-05-20T22:32:25.692 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:25 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.7.service: Deactivated successfully. 2024-05-20T22:32:25.692 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:25 smithi078 systemd[1]: Stopped Ceph osd.7 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:32:25.692 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:25 smithi078 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.7.service: Consumed 7min 34.348s CPU time. 2024-05-20T22:32:25.958 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:25 smithi078 systemd[1]: Starting Ceph osd.7 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:32:25.959 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:25 smithi078 podman[64338]: 2024-05-20 22:32:25.83311664 +0000 UTC m=+0.019612486 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:32:25.959 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:25 smithi078 podman[64338]: 2024-05-20 22:32:25.963691073 +0000 UTC m=+0.150186904 container create 7055ee81eb07b55586f1cdbfda2c81454eebac403c92936067be818666f3978a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate, ceph=True, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:32:26.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 279243 9.91 MB/sec execute 572 sec latency 46.221 ms 2024-05-20T22:32:26.514 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:26 smithi006 ceph-mon[65753]: pgmap v294: 97 pgs: 10 active+undersized+degraded, 23 peering, 3 stale+active+clean, 61 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.3 MiB/s wr, 280 op/s; 40/1650 objects degraded (2.424%) 2024-05-20T22:32:26.514 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:26 smithi006 ceph-mon[65753]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-20T22:32:26.515 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:26 smithi006 ceph-mon[65753]: Health check failed: Degraded data redundancy: 40/1650 objects degraded (2.424%), 10 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:26.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:26 smithi078 ceph-mon[52919]: pgmap v294: 97 pgs: 10 active+undersized+degraded, 23 peering, 3 stale+active+clean, 61 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.3 MiB/s wr, 280 op/s; 40/1650 objects degraded (2.424%) 2024-05-20T22:32:26.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:26 smithi078 ceph-mon[52919]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-05-20T22:32:26.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:26 smithi078 ceph-mon[52919]: Health check failed: Degraded data redundancy: 40/1650 objects degraded (2.424%), 10 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:26.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 podman[64338]: 2024-05-20 22:32:26.27510578 +0000 UTC m=+0.461601625 container init 7055ee81eb07b55586f1cdbfda2c81454eebac403c92936067be818666f3978a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True) 2024-05-20T22:32:26.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 podman[64338]: 2024-05-20 22:32:26.278812827 +0000 UTC m=+0.465308667 container start 7055ee81eb07b55586f1cdbfda2c81454eebac403c92936067be818666f3978a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-05-20T22:32:26.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 podman[64338]: 2024-05-20 22:32:26.375369682 +0000 UTC m=+0.561865515 container attach 7055ee81eb07b55586f1cdbfda2c81454eebac403c92936067be818666f3978a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.schema-version=1.0) 2024-05-20T22:32:27.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 279608 9.91 MB/sec execute 573 sec latency 75.693 ms 2024-05-20T22:32:27.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate[64354]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T22:32:27.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate[64354]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T22:32:27.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate[64354]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 bash[64338]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 bash[64338]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 bash[64338]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 bash[64338]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 bash[64338]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 bash[64338]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 bash[64338]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate[64354]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate[64354]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate[64354]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate[64354]: --> ceph-volume raw activate successful for osd ID: 7 2024-05-20T22:32:27.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:26 smithi078 podman[64338]: 2024-05-20 22:32:26.801410999 +0000 UTC m=+0.987906839 container died 7055ee81eb07b55586f1cdbfda2c81454eebac403c92936067be818666f3978a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:32:27.835 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:27 smithi006 ceph-mon[65753]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-20T22:32:27.849 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:27 smithi078 ceph-mon[52919]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-05-20T22:32:27.849 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:27 smithi078 podman[64469]: 2024-05-20 22:32:27.480721208 +0000 UTC m=+0.672532308 container remove 7055ee81eb07b55586f1cdbfda2c81454eebac403c92936067be818666f3978a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0) 2024-05-20T22:32:27.849 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:27 smithi078 podman[64528]: 2024-05-20 22:32:27.598932898 +0000 UTC m=+0.017609184 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:32:27.849 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:27 smithi078 podman[64528]: 2024-05-20 22:32:27.723591545 +0000 UTC m=+0.142267819 container create 2ec32e6186e13241b162df647c832f4b2a9614179722bcf144de8e7b2f86139a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD) 2024-05-20T22:32:28.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 280007 9.91 MB/sec execute 574 sec latency 38.190 ms 2024-05-20T22:32:28.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:27 smithi078 podman[64528]: 2024-05-20 22:32:27.968207107 +0000 UTC m=+0.386883403 container init 2ec32e6186e13241b162df647c832f4b2a9614179722bcf144de8e7b2f86139a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-20T22:32:28.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:27 smithi078 podman[64528]: 2024-05-20 22:32:27.971580839 +0000 UTC m=+0.390257123 container start 2ec32e6186e13241b162df647c832f4b2a9614179722bcf144de8e7b2f86139a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:32:28.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:27 smithi078 ceph-osd[64547]: -- 172.21.15.78:0/438957425 <== mon.1 v2:172.21.15.78:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5646b3285680 con 0x5646b324f000 2024-05-20T22:32:28.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:28 smithi078 bash[64528]: 2ec32e6186e13241b162df647c832f4b2a9614179722bcf144de8e7b2f86139a 2024-05-20T22:32:28.146 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:28 smithi078 systemd[1]: Started Ceph osd.7 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:32:28.895 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:28 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:32:28.519+0000 7f2f9fe077c0 -1 Falling back to public interface 2024-05-20T22:32:29.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 280292 9.90 MB/sec execute 575 sec latency 56.460 ms 2024-05-20T22:32:29.373 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:29 smithi006 ceph-mon[65753]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 357 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 258/1620 objects degraded (15.926%) 2024-05-20T22:32:29.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:29.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:29.374 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:29.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:29 smithi078 ceph-mon[52919]: pgmap v295: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 357 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 258/1620 objects degraded (15.926%) 2024-05-20T22:32:29.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:29 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:29.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:29 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:29 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:30.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 280608 9.91 MB/sec execute 576 sec latency 45.752 ms 2024-05-20T22:32:30.942 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:32:31.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 280826 9.90 MB/sec execute 577 sec latency 88.201 ms 2024-05-20T22:32:31.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:31 smithi006 ceph-mon[65753]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 356 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 503 op/s; 258/1644 objects degraded (15.693%) 2024-05-20T22:32:31.362 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:31.363 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:31.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:31 smithi078 ceph-mon[52919]: pgmap v296: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 356 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.9 MiB/s wr, 503 op/s; 258/1644 objects degraded (15.693%) 2024-05-20T22:32:31.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:31.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:32.036 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (9m) 2m ago 18m 14.7M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (5m) 2m ago 18m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (5m) 37s ago 17m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (9m) 2m ago 17m 43.8M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (12m) 2m ago 12m 37.6M - 17.2.0 e1d6a67b021e a0587409b054 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (12m) 37s ago 12m 14.1M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (8m) 2m ago 19m 510M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (7m) 37s ago 17m 453M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (7m) 2m ago 19m 52.4M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (6m) 37s ago 17m 44.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (11m) 2m ago 11m 283M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (11m) 37s ago 11m 88.0M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:32:32.037 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (9m) 2m ago 17m 11.5M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (8m) 37s ago 17m 16.9M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (5m) 2m ago 16m 358M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (4m) 2m ago 16m 287M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (3m) 2m ago 15m 196M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (2m) 2m ago 15m 11.8M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (2m) 37s ago 14m 296M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (81s) 37s ago 14m 273M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (43s) 37s ago 14m 12.3M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 starting - - - 2704M 2024-05-20T22:32:32.038 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (7m) 2m ago 16m 68.0M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:32:32.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 281102 9.90 MB/sec execute 578 sec latency 65.882 ms 2024-05-20T22:32:32.377 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:32 smithi006 ceph-mon[65753]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:32.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:32 smithi078 ceph-mon[52919]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:32.678 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 7 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:32:32.679 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:32:32.680 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-05-20T22:32:32.680 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 11 2024-05-20T22:32:32.680 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:32:32.680 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:32:33.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 281386 9.89 MB/sec execute 579 sec latency 57.066 ms 2024-05-20T22:32:33.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:33 smithi006 ceph-mon[65753]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:33.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:33 smithi006 ceph-mon[65753]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 357 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 428 op/s; 256/1659 objects degraded (15.431%) 2024-05-20T22:32:33.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:33 smithi006 ceph-mon[65753]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:33.141 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:33 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 256/1659 objects degraded (15.431%), 35 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:33.141 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:33 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/755663196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:33.141 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:33.141 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:33.243 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:32:33.243 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:32:33.243 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr", 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "osd" 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "14/23 daemons upgraded", 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading osd daemons", 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:32:33.244 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:32:33.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:33 smithi078 ceph-mon[52919]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:33.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:33 smithi078 ceph-mon[52919]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 357 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 428 op/s; 256/1659 objects degraded (15.431%) 2024-05-20T22:32:33.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:33 smithi078 ceph-mon[52919]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:33.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:33 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 256/1659 objects degraded (15.431%), 35 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:33.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:33 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/755663196' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:33.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:33.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:33.895 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 256/1659 objects degraded (15.431%), 35 pgs degraded 2024-05-20T22:32:33.895 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_DOWN: 1 osds down 2024-05-20T22:32:33.896 INFO:teuthology.orchestra.run.smithi006.stdout: osd.7 (root=default,host=smithi078) is down 2024-05-20T22:32:33.896 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-05-20T22:32:33.896 INFO:teuthology.orchestra.run.smithi006.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-05-20T22:32:33.896 INFO:teuthology.orchestra.run.smithi006.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 256/1659 objects degraded (15.431%), 35 pgs degraded 2024-05-20T22:32:33.896 INFO:teuthology.orchestra.run.smithi006.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-05-20T22:32:33.896 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-05-20T22:32:33.896 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-05-20T22:32:33.897 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-05-20T22:32:33.897 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-05-20T22:32:33.897 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-05-20T22:32:33.897 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-05-20T22:32:33.897 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-05-20T22:32:33.897 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-05-20T22:32:33.898 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-05-20T22:32:33.898 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-05-20T22:32:33.898 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-05-20T22:32:33.898 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-05-20T22:32:33.898 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-05-20T22:32:33.898 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-05-20T22:32:33.899 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-05-20T22:32:33.899 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-05-20T22:32:33.899 INFO:teuthology.orchestra.run.smithi006.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-05-20T22:32:33.899 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-05-20T22:32:33.899 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-05-20T22:32:33.899 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-05-20T22:32:33.899 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-05-20T22:32:33.900 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-05-20T22:32:33.900 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-05-20T22:32:33.900 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-05-20T22:32:33.900 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-05-20T22:32:33.900 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-05-20T22:32:33.900 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-05-20T22:32:33.900 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-05-20T22:32:33.901 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-05-20T22:32:33.901 INFO:teuthology.orchestra.run.smithi006.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-05-20T22:32:33.901 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-05-20T22:32:33.901 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-05-20T22:32:33.901 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-05-20T22:32:33.901 INFO:teuthology.orchestra.run.smithi006.stdout: pg 4.14 is active+undersized+degraded, acting [3,2] 2024-05-20T22:32:34.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 281628 9.88 MB/sec execute 580 sec latency 46.387 ms 2024-05-20T22:32:34.174 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:34 smithi006 ceph-mon[65753]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:34.174 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:34 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2633600542' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:32:34.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:34 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:34.175 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:34 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:34.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:34 smithi078 ceph-mon[52919]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:32:34.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:34 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2633600542' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:32:34.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:34 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:34.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:34 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:35.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 281944 9.88 MB/sec execute 581 sec latency 46.165 ms 2024-05-20T22:32:35.186 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:35 smithi006 ceph-mon[65753]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 398 op/s; 256/1659 objects degraded (15.431%) 2024-05-20T22:32:35.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:35 smithi078 ceph-mon[52919]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 354 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.3 MiB/s wr, 398 op/s; 256/1659 objects degraded (15.431%) 2024-05-20T22:32:36.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 282402 9.88 MB/sec execute 582 sec latency 30.709 ms 2024-05-20T22:32:37.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 282761 9.88 MB/sec execute 583 sec latency 58.934 ms 2024-05-20T22:32:37.242 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:37 smithi006 ceph-mon[65753]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.7 MiB/s wr, 507 op/s; 257/1635 objects degraded (15.719%) 2024-05-20T22:32:37.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:37 smithi078 ceph-mon[52919]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.7 MiB/s wr, 507 op/s; 257/1635 objects degraded (15.719%) 2024-05-20T22:32:38.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 283113 9.89 MB/sec execute 584 sec latency 41.813 ms 2024-05-20T22:32:39.117 INFO:teuthology.orchestra.run.smithi006.stdout: 5 283404 9.88 MB/sec execute 585 sec latency 54.611 ms 2024-05-20T22:32:39.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.6 MiB/s wr, 553 op/s; 246/1608 objects degraded (15.299%) 2024-05-20T22:32:39.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:39.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:32:39.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 246/1608 objects degraded (15.299%), 35 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all osd 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-20T22:32:39.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-20T22:32:39.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-20T22:32:39.270 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-20T22:32:39.270 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: Upgrade: Setting require_osd_release to 19 squid 2024-05-20T22:32:39.270 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-20T22:32:39.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.6 MiB/s wr, 553 op/s; 246/1608 objects degraded (15.299%) 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 246/1608 objects degraded (15.299%), 35 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all osd 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-05-20T22:32:39.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: Upgrade: Setting require_osd_release to 19 squid 2024-05-20T22:32:39.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-05-20T22:32:39.895 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:39 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:32:39.515+0000 7f2f9fe077c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-05-20T22:32:40.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 283751 9.88 MB/sec execute 586 sec latency 58.423 ms 2024-05-20T22:32:40.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:40 smithi006 ceph-mon[65753]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-20T22:32:40.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:40 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-20T22:32:40.281 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:40 smithi006 ceph-mon[65753]: osdmap e149: 8 total, 7 up, 8 in 2024-05-20T22:32:40.282 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:40 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi006.pburlb"]}]: dispatch 2024-05-20T22:32:40.282 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:40 smithi006 ceph-mon[65753]: Upgrade: It appears safe to stop mds.foofs.smithi006.pburlb 2024-05-20T22:32:40.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:40 smithi078 ceph-mon[52919]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-05-20T22:32:40.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:40 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-05-20T22:32:40.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:40 smithi078 ceph-mon[52919]: osdmap e149: 8 total, 7 up, 8 in 2024-05-20T22:32:40.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:40 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi006.pburlb"]}]: dispatch 2024-05-20T22:32:40.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:40 smithi078 ceph-mon[52919]: Upgrade: It appears safe to stop mds.foofs.smithi006.pburlb 2024-05-20T22:32:41.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 283986 9.87 MB/sec execute 587 sec latency 61.468 ms 2024-05-20T22:32:41.545 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:41 smithi006 ceph-mon[65753]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 321 op/s; 245/1617 objects degraded (15.152%) 2024-05-20T22:32:41.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:41 smithi078 ceph-mon[52919]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.0 MiB/s wr, 321 op/s; 245/1617 objects degraded (15.152%) 2024-05-20T22:32:41.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:41 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:32:41.396+0000 7f2f9fe077c0 -1 osd.7 146 log_to_monitors true 2024-05-20T22:32:42.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284301 9.88 MB/sec execute 588 sec latency 59.767 ms 2024-05-20T22:32:42.302 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:42 smithi006 ceph-mon[65753]: from='osd.7 [v2:172.21.15.78:6824/1555117355,v1:172.21.15.78:6825/1555117355]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:32:42.302 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:42 smithi006 ceph-mon[65753]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:32:42.302 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:42.303 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:32:42.303 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:42.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:42 smithi078 ceph-mon[52919]: from='osd.7 [v2:172.21.15.78:6824/1555117355,v1:172.21.15.78:6825/1555117355]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:32:42.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:42 smithi078 ceph-mon[52919]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-05-20T22:32:42.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:42.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi006.pburlb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:32:42.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:32:43.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284559 9.87 MB/sec execute 589 sec latency 123.118 ms 2024-05-20T22:32:43.145 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:42 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:32:42.829+0000 7f2f8d256700 -1 osd.7 146 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-05-20T22:32:43.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 486 op/s; 249/1635 objects degraded (15.229%) 2024-05-20T22:32:43.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: Upgrade: Updating mds.foofs.smithi006.pburlb 2024-05-20T22:32:43.327 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: Deploying daemon mds.foofs.smithi006.pburlb on smithi006 2024-05-20T22:32:43.328 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T22:32:43.328 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: from='osd.7 [v2:172.21.15.78:6824/1555117355,v1:172.21.15.78:6825/1555117355]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:43.328 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: osdmap e150: 8 total, 7 up, 8 in 2024-05-20T22:32:43.328 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:43.328 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:43 smithi006 ceph-mon[65753]: from='osd.7 ' entity='osd.7' 2024-05-20T22:32:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: pgmap v303: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 486 op/s; 249/1635 objects degraded (15.229%) 2024-05-20T22:32:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: Upgrade: Updating mds.foofs.smithi006.pburlb 2024-05-20T22:32:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: Deploying daemon mds.foofs.smithi006.pburlb on smithi006 2024-05-20T22:32:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-05-20T22:32:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: from='osd.7 [v2:172.21.15.78:6824/1555117355,v1:172.21.15.78:6825/1555117355]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: osdmap e150: 8 total, 7 up, 8 in 2024-05-20T22:32:43.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi078", "root=default"]}]: dispatch 2024-05-20T22:32:43.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:43 smithi078 ceph-mon[52919]: from='osd.7 ' entity='osd.7' 2024-05-20T22:32:44.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284830 9.86 MB/sec execute 590 sec latency 313.104 ms 2024-05-20T22:32:44.513 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:44 smithi006 ceph-mon[65753]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:32:44.513 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:44 smithi006 ceph-mon[65753]: osd.7 [v2:172.21.15.78:6824/1555117355,v1:172.21.15.78:6825/1555117355] boot 2024-05-20T22:32:44.513 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:44 smithi006 ceph-mon[65753]: osdmap e151: 8 total, 8 up, 8 in 2024-05-20T22:32:44.513 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:44 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:32:44.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:44 smithi078 ceph-mon[52919]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-05-20T22:32:44.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:44 smithi078 ceph-mon[52919]: osd.7 [v2:172.21.15.78:6824/1555117355,v1:172.21.15.78:6825/1555117355] boot 2024-05-20T22:32:44.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:44 smithi078 ceph-mon[52919]: osdmap e151: 8 total, 8 up, 8 in 2024-05-20T22:32:44.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:44 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-05-20T22:32:45.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284830 9.85 MB/sec execute 591 sec latency 1313.138 ms 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[65749]: 2024-05-20T22:32:45.177+0000 7f9825232700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: pgmap v305: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.1 MiB/s wr, 446 op/s; 257/1638 objects degraded (15.690%) 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 257/1638 objects degraded (15.690%), 35 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: osdmap e152: 8 total, 8 up, 8 in 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: osdmap e153: 8 total, 8 up, 8 in 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: Standby daemon mds.foofs.smithi078.yocghn assigned to filesystem foofs as rank 0 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-05-20T22:32:45.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:45 smithi006 ceph-mon[65753]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:replay} 2024-05-20T22:32:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: pgmap v305: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 352 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.1 MiB/s wr, 446 op/s; 257/1638 objects degraded (15.690%) 2024-05-20T22:32:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 257/1638 objects degraded (15.690%), 35 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: osdmap e152: 8 total, 8 up, 8 in 2024-05-20T22:32:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-05-20T22:32:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:32:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: osdmap e153: 8 total, 8 up, 8 in 2024-05-20T22:32:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: Standby daemon mds.foofs.smithi078.yocghn assigned to filesystem foofs as rank 0 2024-05-20T22:32:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-05-20T22:32:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T22:32:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-05-20T22:32:45.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:45 smithi078 ceph-mon[52919]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:replay} 2024-05-20T22:32:45.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:45 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:32:45.264+0000 7f2f77a2b700 -1 bdev(0x5646b404a000 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 3 2024-05-20T22:32:45.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:45 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:32:45.265+0000 7f2f7822c700 -1 bdev(0x5646b404a000 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 2 2024-05-20T22:32:45.646 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:32:45 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:32:45.266+0000 7f2f7922e700 -1 bdev(0x5646b404a000 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 8 2024-05-20T22:32:46.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284830 9.83 MB/sec execute 592 sec latency 2313.174 ms 2024-05-20T22:32:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:46 smithi006 ceph-mon[65753]: pgmap v309: 97 pgs: 8 peering, 7 active+undersized, 27 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.1 MiB/s wr, 340 op/s; 223/1602 objects degraded (13.920%) 2024-05-20T22:32:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:46 smithi006 ceph-mon[65753]: osdmap e154: 8 total, 8 up, 8 in 2024-05-20T22:32:46.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:32:46.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:46.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:32:46.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:46 smithi078 ceph-mon[52919]: pgmap v309: 97 pgs: 8 peering, 7 active+undersized, 27 active+undersized+degraded, 55 active+clean; 351 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.1 MiB/s wr, 340 op/s; 223/1602 objects degraded (13.920%) 2024-05-20T22:32:46.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:46 smithi078 ceph-mon[52919]: osdmap e154: 8 total, 8 up, 8 in 2024-05-20T22:32:46.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:32:47.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284830 9.81 MB/sec execute 593 sec latency 3313.209 ms 2024-05-20T22:32:48.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:47 smithi006 ceph-mon[65753]: osdmap e155: 8 total, 8 up, 8 in 2024-05-20T22:32:48.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284830 9.80 MB/sec execute 594 sec latency 4313.244 ms 2024-05-20T22:32:48.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:47 smithi078 ceph-mon[52919]: osdmap e155: 8 total, 8 up, 8 in 2024-05-20T22:32:49.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:48 smithi006 ceph-mon[65753]: pgmap v312: 97 pgs: 12 peering, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.4 MiB/s wr, 327 op/s; 56/1635 objects degraded (3.425%); 2.8 MiB/s, 47 objects/s recovering 2024-05-20T22:32:49.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:48 smithi006 ceph-mon[65753]: reconnect by client.24375 172.21.15.78:0/2587796291 after 0.000999984 2024-05-20T22:32:49.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:48 smithi006 ceph-mon[65753]: reconnect by client.14536 172.21.15.6:0/1938864191 after 0.000999984 2024-05-20T22:32:49.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:48 smithi006 ceph-mon[65753]: osdmap e156: 8 total, 8 up, 8 in 2024-05-20T22:32:49.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:48 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:reconnect 2024-05-20T22:32:49.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:48 smithi006 ceph-mon[65753]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:reconnect} 2024-05-20T22:32:49.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284830 9.78 MB/sec execute 595 sec latency 5313.287 ms 2024-05-20T22:32:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:48 smithi078 ceph-mon[52919]: pgmap v312: 97 pgs: 12 peering, 2 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.4 MiB/s wr, 327 op/s; 56/1635 objects degraded (3.425%); 2.8 MiB/s, 47 objects/s recovering 2024-05-20T22:32:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:48 smithi078 ceph-mon[52919]: reconnect by client.24375 172.21.15.78:0/2587796291 after 0.000999984 2024-05-20T22:32:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:48 smithi078 ceph-mon[52919]: reconnect by client.14536 172.21.15.6:0/1938864191 after 0.000999984 2024-05-20T22:32:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:48 smithi078 ceph-mon[52919]: osdmap e156: 8 total, 8 up, 8 in 2024-05-20T22:32:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:48 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:reconnect 2024-05-20T22:32:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:48 smithi078 ceph-mon[52919]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:reconnect} 2024-05-20T22:32:50.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:49 smithi006 ceph-mon[65753]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-05-20T22:32:50.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:49 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:rejoin 2024-05-20T22:32:50.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:49 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.6:6834/3542901344,v1:172.21.15.6:6835/3542901344] up:boot 2024-05-20T22:32:50.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:49 smithi006 ceph-mon[65753]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:rejoin} 1 up:standby 2024-05-20T22:32:50.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:49 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:32:50.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 284830 9.76 MB/sec execute 596 sec latency 6313.317 ms 2024-05-20T22:32:50.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:49 smithi078 ceph-mon[52919]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-05-20T22:32:50.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:49 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:rejoin 2024-05-20T22:32:50.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:49 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.6:6834/3542901344,v1:172.21.15.6:6835/3542901344] up:boot 2024-05-20T22:32:50.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:49 smithi078 ceph-mon[52919]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:rejoin} 1 up:standby 2024-05-20T22:32:50.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:49 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:32:50.918 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:50 smithi006 ceph-mon[65753]: pgmap v314: 97 pgs: 8 peering, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s rd, 1.4 MiB/s wr, 180 op/s; 56/1635 objects degraded (3.425%); 3.3 MiB/s, 40 objects/s recovering 2024-05-20T22:32:50.918 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:50 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:clientreplay 2024-05-20T22:32:50.918 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:50 smithi006 ceph-mon[65753]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:clientreplay} 1 up:standby 2024-05-20T22:32:50.919 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:50 smithi006 ceph-mon[65753]: daemon mds.foofs.smithi078.yocghn is now active in filesystem foofs as rank 0 2024-05-20T22:32:50.919 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:50 smithi006 ceph-mon[65753]: Health check update: Degraded data redundancy: 56/1635 objects degraded (3.425%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:51.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 285024 9.75 MB/sec execute 597 sec latency 6862.992 ms 2024-05-20T22:32:51.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:50 smithi078 ceph-mon[52919]: pgmap v314: 97 pgs: 8 peering, 2 active+recovery_wait+undersized+degraded+remapped, 87 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s rd, 1.4 MiB/s wr, 180 op/s; 56/1635 objects degraded (3.425%); 3.3 MiB/s, 40 objects/s recovering 2024-05-20T22:32:51.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:50 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:clientreplay 2024-05-20T22:32:51.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:50 smithi078 ceph-mon[52919]: fsmap foofs:1/1 {0=foofs.smithi078.yocghn=up:clientreplay} 1 up:standby 2024-05-20T22:32:51.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:50 smithi078 ceph-mon[52919]: daemon mds.foofs.smithi078.yocghn is now active in filesystem foofs as rank 0 2024-05-20T22:32:51.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:50 smithi078 ceph-mon[52919]: Health check update: Degraded data redundancy: 56/1635 objects degraded (3.425%), 2 pgs degraded (PG_DEGRADED) 2024-05-20T22:32:51.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:51 smithi078 ceph-mon[52919]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-05-20T22:32:51.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:51 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:active 2024-05-20T22:32:51.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:51 smithi078 ceph-mon[52919]: fsmap foofs:1 {0=foofs.smithi078.yocghn=up:active} 1 up:standby 2024-05-20T22:32:51.923 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:51 smithi006 ceph-mon[65753]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-05-20T22:32:51.923 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:51 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.78:6832/583853724,v1:172.21.15.78:6833/583853724] up:active 2024-05-20T22:32:51.923 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:51 smithi006 ceph-mon[65753]: fsmap foofs:1 {0=foofs.smithi078.yocghn=up:active} 1 up:standby 2024-05-20T22:32:52.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 285362 9.75 MB/sec execute 598 sec latency 39.460 ms 2024-05-20T22:32:52.688 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:52 smithi006 ceph-mon[65753]: pgmap v315: 97 pgs: 4 peering, 93 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 1.0 MiB/s wr, 143 op/s; 4.0 MiB/s, 39 objects/s recovering 2024-05-20T22:32:52.688 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:52 smithi006 ceph-mon[65753]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/1635 objects degraded (3.425%), 2 pgs degraded) 2024-05-20T22:32:52.688 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:52 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:32:53.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:52 smithi078 ceph-mon[52919]: pgmap v315: 97 pgs: 4 peering, 93 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 1.0 MiB/s wr, 143 op/s; 4.0 MiB/s, 39 objects/s recovering 2024-05-20T22:32:53.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:52 smithi078 ceph-mon[52919]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/1635 objects degraded (3.425%), 2 pgs degraded) 2024-05-20T22:32:53.103 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:52 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:32:53.118 INFO:teuthology.orchestra.run.smithi006.stdout: 5 285762 9.75 MB/sec execute 599 sec latency 71.895 ms 2024-05-20T22:32:54.119 INFO:teuthology.orchestra.run.smithi006.stdout: 5 cleanup 600 sec 2024-05-20T22:32:54.406 INFO:teuthology.orchestra.run.smithi006.stdout: 0 cleanup 600 sec 2024-05-20T22:32:54.406 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:32:54.406 INFO:teuthology.orchestra.run.smithi006.stdout: Operation Count AvgLat MaxLat 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: ---------------------------------------- 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: NTCreateX 187829 4.588 6854.351 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Close 138000 0.146 7.490 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Rename 7953 10.325 6862.980 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Unlink 37897 6.700 2206.029 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Qpathinfo 170421 1.264 6852.498 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Qfileinfo 29748 0.007 3.315 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Qfsinfo 31181 0.650 48.774 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Sfileinfo 15278 12.834 2206.045 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: Find 65830 1.055 22.104 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: WriteX 92829 13.717 6860.772 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: ReadX 294914 0.011 4.182 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: LockX 614 0.408 7.593 2024-05-20T22:32:54.407 INFO:teuthology.orchestra.run.smithi006.stdout: UnlockX 614 0.344 5.233 2024-05-20T22:32:54.408 INFO:teuthology.orchestra.run.smithi006.stdout: Flush 13150 0.003 0.030 2024-05-20T22:32:54.408 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:32:54.408 INFO:teuthology.orchestra.run.smithi006.stdout:Throughput 9.75082 MB/sec 5 clients 5 procs max_latency=6862.992 ms 2024-05-20T22:32:54.409 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-05-20T22:32:54.450 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-05-20T22:32:54.559 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-05-20T22:32:54.585 INFO:teuthology.orchestra.run.smithi006.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-05-20T22:32:54.585 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:32:54.585 INFO:teuthology.orchestra.run.smithi006.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-05-20T22:32:54.586 INFO:teuthology.orchestra.run.smithi006.stdout:3 of 5 processes prepared for launch 0 sec 2024-05-20T22:32:54.686 INFO:teuthology.orchestra.run.smithi006.stdout:5 of 5 processes prepared for launch 0 sec 2024-05-20T22:32:54.686 INFO:teuthology.orchestra.run.smithi006.stdout:releasing clients 2024-05-20T22:32:54.763 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:54 smithi006 ceph-mon[65753]: pgmap v316: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 1.3 MiB/s wr, 144 op/s; 3.2 MiB/s, 42 objects/s recovering 2024-05-20T22:32:55.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:54 smithi078 ceph-mon[52919]: pgmap v316: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 1.3 MiB/s wr, 144 op/s; 3.2 MiB/s, 42 objects/s recovering 2024-05-20T22:32:56.686 INFO:teuthology.orchestra.run.smithi006.stdout: 5 361 22.11 MB/sec execute 1 sec latency 215.998 ms 2024-05-20T22:32:56.842 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:56 smithi006 ceph-mon[65753]: pgmap v317: 97 pgs: 97 active+clean; 357 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.6 MiB/s wr, 280 op/s; 2.8 MiB/s, 35 objects/s recovering 2024-05-20T22:32:57.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:56 smithi078 ceph-mon[52919]: pgmap v317: 97 pgs: 97 active+clean; 357 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 1.6 MiB/s wr, 280 op/s; 2.8 MiB/s, 35 objects/s recovering 2024-05-20T22:32:57.686 INFO:teuthology.orchestra.run.smithi006.stdout: 5 673 31.53 MB/sec execute 2 sec latency 34.529 ms 2024-05-20T22:32:58.686 INFO:teuthology.orchestra.run.smithi006.stdout: 5 1244 25.19 MB/sec execute 3 sec latency 23.369 ms 2024-05-20T22:32:58.866 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:58 smithi006 ceph-mon[65753]: pgmap v318: 97 pgs: 97 active+clean; 356 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.6 MiB/s wr, 377 op/s; 1.5 MiB/s, 16 objects/s recovering 2024-05-20T22:32:58.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:58 smithi006 ceph-mon[65753]: daemon mds.foofs.smithi006.pburlb restarted 2024-05-20T22:32:58.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:58 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:58.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:58 smithi006 ceph-mon[65753]: osdmap e157: 8 total, 8 up, 8 in 2024-05-20T22:32:58.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:58 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:58.867 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:58 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:58 smithi078 ceph-mon[52919]: pgmap v318: 97 pgs: 97 active+clean; 356 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 8.6 MiB/s wr, 377 op/s; 1.5 MiB/s, 16 objects/s recovering 2024-05-20T22:32:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:58 smithi078 ceph-mon[52919]: daemon mds.foofs.smithi006.pburlb restarted 2024-05-20T22:32:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:58 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:58 smithi078 ceph-mon[52919]: osdmap e157: 8 total, 8 up, 8 in 2024-05-20T22:32:59.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:58 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:32:59.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:58 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:32:59.686 INFO:teuthology.orchestra.run.smithi006.stdout: 5 1964 22.46 MB/sec execute 4 sec latency 20.742 ms 2024-05-20T22:32:59.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:59 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:boot 2024-05-20T22:32:59.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:59 smithi006 ceph-mon[65753]: fsmap foofs:1 {0=foofs.smithi078.yocghn=up:active} 1 up:standby 2024-05-20T22:32:59.879 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:32:59 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:33:00.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:59 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:boot 2024-05-20T22:33:00.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:59 smithi078 ceph-mon[52919]: fsmap foofs:1 {0=foofs.smithi078.yocghn=up:active} 1 up:standby 2024-05-20T22:33:00.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:32:59 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:33:00.687 INFO:teuthology.orchestra.run.smithi006.stdout: 5 cleanup 5 sec 2024-05-20T22:33:00.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:00 smithi006 ceph-mon[65753]: pgmap v320: 97 pgs: 97 active+clean; 352 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.5 MiB/s wr, 390 op/s; 1.1 MiB/s, 15 objects/s recovering 2024-05-20T22:33:00.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:00.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:00.896 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:00.924 INFO:teuthology.orchestra.run.smithi006.stdout: 0 cleanup 5 sec 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Operation Count AvgLat MaxLat 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: ---------------------------------------- 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: NTCreateX 1864 3.992 65.348 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Close 1637 3.233 57.280 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Rename 66 7.102 32.167 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Unlink 166 3.654 21.675 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Qpathinfo 1587 1.698 27.057 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Qfileinfo 547 8.808 75.930 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Qfsinfo 196 0.779 5.392 2024-05-20T22:33:00.925 INFO:teuthology.orchestra.run.smithi006.stdout: Sfileinfo 212 7.620 43.920 2024-05-20T22:33:00.926 INFO:teuthology.orchestra.run.smithi006.stdout: Find 535 1.291 4.211 2024-05-20T22:33:00.926 INFO:teuthology.orchestra.run.smithi006.stdout: WriteX 2103 0.023 0.084 2024-05-20T22:33:00.926 INFO:teuthology.orchestra.run.smithi006.stdout: ReadX 1904 0.014 0.584 2024-05-20T22:33:00.926 INFO:teuthology.orchestra.run.smithi006.stdout: Flush 162 7.221 22.467 2024-05-20T22:33:00.926 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:33:00.926 INFO:teuthology.orchestra.run.smithi006.stdout:Throughput 22.4593 MB/sec 5 clients 5 procs max_latency=215.998 ms 2024-05-20T22:33:00.927 DEBUG:teuthology.parallel:result is None 2024-05-20T22:33:01.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:00 smithi078 ceph-mon[52919]: pgmap v320: 97 pgs: 97 active+clean; 352 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.5 MiB/s wr, 390 op/s; 1.1 MiB/s, 15 objects/s recovering 2024-05-20T22:33:01.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:01.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:01.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:02.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:02 smithi006 ceph-mon[65753]: pgmap v321: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 11 MiB/s wr, 421 op/s; 158 KiB/s, 10 objects/s recovering 2024-05-20T22:33:03.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:02 smithi078 ceph-mon[52919]: pgmap v321: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 40 KiB/s rd, 11 MiB/s wr, 421 op/s; 158 KiB/s, 10 objects/s recovering 2024-05-20T22:33:04.274 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (9m) 0s ago 18m 23.6M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (6m) 0s ago 18m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (6m) 32s ago 17m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (9m) 0s ago 18m 45.3M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (7s) 0s ago 12m 12.5M - 19.0.0-2695-g8a373da3 86045c3f4e75 6b776336a4ae 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (12m) 32s ago 12m 14.3M - 17.2.0 e1d6a67b021e 266a40825a7e 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (9m) 0s ago 19m 514M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (8m) 32s ago 17m 453M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:33:04.983 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 0s ago 20m 60.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (7m) 32s ago 17m 45.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (12m) 0s ago 12m 376M - 4.0 e1d6a67b021e c09c7f15ac4f 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (12m) 32s ago 12m 88.1M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (9m) 0s ago 18m 11.8M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 32s ago 17m 16.7M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (5m) 0s ago 17m 461M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (5m) 0s ago 16m 411M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (4m) 0s ago 16m 374M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (3m) 0s ago 15m 360M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (2m) 32s ago 15m 334M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (114s) 32s ago 15m 294M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (76s) 32s ago 14m 278M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (37s) 32s ago 14m 12.3M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 2ec32e6186e1 2024-05-20T22:33:04.984 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (8m) 0s ago 17m 70.4M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:33:04.985 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:04 smithi006 ceph-mon[65753]: pgmap v322: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 14 MiB/s wr, 602 op/s 2024-05-20T22:33:04.985 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:04 smithi006 ceph-mon[65753]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:05.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:04 smithi078 ceph-mon[52919]: pgmap v322: 97 pgs: 97 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 14 MiB/s wr, 602 op/s 2024-05-20T22:33:05.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:04 smithi078 ceph-mon[52919]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:05.431 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:33:05.431 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:33:05.431 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:33:05.431 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:05.431 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:33:05.431 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:33:05.431 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 1 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 13 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:33:05.432 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr", 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "osd" 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "15/23 daemons upgraded", 2024-05-20T22:33:05.791 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading mds daemons", 2024-05-20T22:33:05.792 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:33:05.792 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:33:06.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:05 smithi078 ceph-mon[52919]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:06.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:06.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:06.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:05 smithi078 ceph-mon[52919]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:06.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:05 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/683017746' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:06.224 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:33:06.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:05 smithi006 ceph-mon[65753]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:06.224 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:06.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:06.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:05 smithi006 ceph-mon[65753]: from='client.34502 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:06.225 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:05 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/683017746' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:07.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:07 smithi006 ceph-mon[65753]: pgmap v323: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 503 op/s 2024-05-20T22:33:07.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:07 smithi006 ceph-mon[65753]: from='client.34510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:07.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:07 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:07.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:07 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:07.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:07 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/907520945' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:33:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:07 smithi078 ceph-mon[52919]: pgmap v323: 97 pgs: 97 active+clean; 330 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 14 MiB/s wr, 503 op/s 2024-05-20T22:33:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:07 smithi078 ceph-mon[52919]: from='client.34510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:07 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:07 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:07 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/907520945' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:33:09.096 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:09 smithi006 ceph-mon[65753]: pgmap v324: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 343 op/s 2024-05-20T22:33:09.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:09 smithi078 ceph-mon[52919]: pgmap v324: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 343 op/s 2024-05-20T22:33:10.875 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 296 op/s 2024-05-20T22:33:10.875 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: Detected new or changed devices on smithi006 2024-05-20T22:33:10.875 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.yocghn"]}]: dispatch 2024-05-20T22:33:10.876 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:10 smithi078 ceph-mon[52919]: Upgrade: It appears safe to stop mds.foofs.smithi078.yocghn 2024-05-20T22:33:11.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: pgmap v325: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.4 MiB/s wr, 296 op/s 2024-05-20T22:33:11.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: Detected new or changed devices on smithi006 2024-05-20T22:33:11.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:11.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi078.yocghn"]}]: dispatch 2024-05-20T22:33:11.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:10 smithi006 ceph-mon[65753]: Upgrade: It appears safe to stop mds.foofs.smithi078.yocghn 2024-05-20T22:33:12.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:12 smithi078 ceph-mon[52919]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.5 MiB/s wr, 274 op/s 2024-05-20T22:33:12.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:12 smithi078 ceph-mon[52919]: Upgrade: Updating mds.foofs.smithi078.yocghn 2024-05-20T22:33:12.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:12 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:12.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:12 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:33:12.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:12 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:12.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:12 smithi078 ceph-mon[52919]: Deploying daemon mds.foofs.smithi078.yocghn on smithi078 2024-05-20T22:33:13.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:12 smithi006 ceph-mon[65753]: pgmap v326: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.5 MiB/s wr, 274 op/s 2024-05-20T22:33:13.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:12 smithi006 ceph-mon[65753]: Upgrade: Updating mds.foofs.smithi078.yocghn 2024-05-20T22:33:13.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:12 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:13.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:12 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi078.yocghn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-05-20T22:33:13.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:12 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:13.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:12 smithi006 ceph-mon[65753]: Deploying daemon mds.foofs.smithi078.yocghn on smithi078 2024-05-20T22:33:14.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:13 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[65749]: 2024-05-20T22:33:13.961+0000 7f9825232700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:33:15.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.1 MiB/s wr, 243 op/s 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: osdmap e158: 8 total, 8 up, 8 in 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: Standby daemon mds.foofs.smithi006.pburlb assigned to filesystem foofs as rank 0 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-05-20T22:33:15.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:14 smithi006 ceph-mon[65753]: fsmap foofs:1/1 {0=foofs.smithi006.pburlb=up:replay} 2024-05-20T22:33:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: pgmap v327: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.1 MiB/s wr, 243 op/s 2024-05-20T22:33:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-05-20T22:33:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-05-20T22:33:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: osdmap e158: 8 total, 8 up, 8 in 2024-05-20T22:33:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: Standby daemon mds.foofs.smithi006.pburlb assigned to filesystem foofs as rank 0 2024-05-20T22:33:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-05-20T22:33:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-05-20T22:33:15.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-05-20T22:33:15.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:14 smithi078 ceph-mon[52919]: fsmap foofs:1/1 {0=foofs.smithi006.pburlb=up:replay} 2024-05-20T22:33:16.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:16.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:16.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 21 KiB/s wr, 35 op/s 2024-05-20T22:33:16.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: reconnect by client.14536 172.21.15.6:0/1938864191 after 0 2024-05-20T22:33:16.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:reconnect 2024-05-20T22:33:16.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: fsmap foofs:1/1 {0=foofs.smithi006.pburlb=up:reconnect} 2024-05-20T22:33:16.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: reconnect by client.24375 172.21.15.78:0/2587796291 after 0.0299997 2024-05-20T22:33:16.785 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:33:16.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:16.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:16.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: pgmap v329: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 21 KiB/s wr, 35 op/s 2024-05-20T22:33:16.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: reconnect by client.14536 172.21.15.6:0/1938864191 after 0 2024-05-20T22:33:16.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:reconnect 2024-05-20T22:33:16.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: fsmap foofs:1/1 {0=foofs.smithi006.pburlb=up:reconnect} 2024-05-20T22:33:16.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: reconnect by client.24375 172.21.15.78:0/2587796291 after 0.0299997 2024-05-20T22:33:16.896 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:33:18.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:17 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:rejoin 2024-05-20T22:33:18.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:17 smithi006 ceph-mon[65753]: fsmap foofs:1/1 {0=foofs.smithi006.pburlb=up:rejoin} 2024-05-20T22:33:18.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:17 smithi006 ceph-mon[65753]: daemon mds.foofs.smithi006.pburlb is now active in filesystem foofs as rank 0 2024-05-20T22:33:18.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:17 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:rejoin 2024-05-20T22:33:18.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:17 smithi078 ceph-mon[52919]: fsmap foofs:1/1 {0=foofs.smithi006.pburlb=up:rejoin} 2024-05-20T22:33:18.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:17 smithi078 ceph-mon[52919]: daemon mds.foofs.smithi006.pburlb is now active in filesystem foofs as rank 0 2024-05-20T22:33:19.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:18 smithi006 ceph-mon[65753]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 21 KiB/s wr, 8 op/s 2024-05-20T22:33:19.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:18 smithi006 ceph-mon[65753]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-05-20T22:33:19.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:18 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:active 2024-05-20T22:33:19.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:18 smithi006 ceph-mon[65753]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 2024-05-20T22:33:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:18 smithi078 ceph-mon[52919]: pgmap v330: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 21 KiB/s wr, 8 op/s 2024-05-20T22:33:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:18 smithi078 ceph-mon[52919]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-05-20T22:33:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:18 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418] up:active 2024-05-20T22:33:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:18 smithi078 ceph-mon[52919]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 2024-05-20T22:33:21.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:20 smithi006 ceph-mon[65753]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 21 KiB/s wr, 10 op/s 2024-05-20T22:33:21.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:20 smithi078 ceph-mon[52919]: pgmap v331: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 21 KiB/s wr, 10 op/s 2024-05-20T22:33:22.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:22 smithi078 ceph-mon[52919]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T22:33:22.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:22 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:22.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:22 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:22.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:22 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:23.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:22 smithi006 ceph-mon[65753]: pgmap v332: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T22:33:23.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:22 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:23.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:22 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:23.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:22 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:24.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:23 smithi006 ceph-mon[65753]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-05-20T22:33:24.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:23 smithi006 ceph-mon[65753]: Cluster is now healthy 2024-05-20T22:33:24.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:23 smithi006 ceph-mon[65753]: mds.? [v2:172.21.15.78:6832/2840025487,v1:172.21.15.78:6833/2840025487] up:boot 2024-05-20T22:33:24.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:23 smithi006 ceph-mon[65753]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:33:24.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:23 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:33:24.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:23 smithi078 ceph-mon[52919]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-05-20T22:33:24.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:23 smithi078 ceph-mon[52919]: Cluster is now healthy 2024-05-20T22:33:24.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:23 smithi078 ceph-mon[52919]: mds.? [v2:172.21.15.78:6832/2840025487,v1:172.21.15.78:6833/2840025487] up:boot 2024-05-20T22:33:24.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:23 smithi078 ceph-mon[52919]: fsmap foofs:1 {0=foofs.smithi006.pburlb=up:active} 1 up:standby 2024-05-20T22:33:24.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:23 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:33:24.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:24 smithi078 ceph-mon[52919]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T22:33:25.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:24 smithi006 ceph-mon[65753]: pgmap v333: 97 pgs: 97 active+clean; 236 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 921 B/s wr, 11 op/s 2024-05-20T22:33:26.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:26 smithi078 ceph-mon[52919]: pgmap v334: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.1 KiB/s wr, 13 op/s 2024-05-20T22:33:26.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:26.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:26 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:27.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:26 smithi006 ceph-mon[65753]: pgmap v334: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.1 KiB/s wr, 13 op/s 2024-05-20T22:33:27.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:27.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:26 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:27.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:27 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:27.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:27 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:28.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:28.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:27 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:28.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:28 smithi078 ceph-mon[52919]: pgmap v335: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-05-20T22:33:29.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:28 smithi006 ceph-mon[65753]: pgmap v335: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-05-20T22:33:31.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: pgmap v336: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-05-20T22:33:31.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: Detected new or changed devices on smithi078 2024-05-20T22:33:31.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:31.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:31.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all mds 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.pburlb"}]': finished 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:33:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.yocghn"}]': finished 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all rgw 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all iscsi 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.270 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:31.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: pgmap v336: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: Detected new or changed devices on smithi078 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all mds 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.pburlb"}]: dispatch 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi006.pburlb"}]': finished 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.yocghn"}]: dispatch 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi078.yocghn"}]': finished 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.647 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all rgw 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all rbd-mirror 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all ceph-exporter 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all iscsi 2024-05-20T22:33:31.648 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:31.649 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:33.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: pgmap v337: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-05-20T22:33:33.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: Upgrade: Updating nfs.foo.0.0.smithi006.ecmyeo (1/2) 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: Rados config object exists: conf-nfs.foo 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo-rgw 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:33.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: Bind address in nfs.foo.0.0.smithi006.ecmyeo's ganesha conf is defaulting to empty 2024-05-20T22:33:33.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:33.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:33 smithi006 ceph-mon[65753]: Deploying daemon nfs.foo.0.0.smithi006.ecmyeo on smithi006 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: pgmap v337: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: Upgrade: Updating nfs.foo.0.0.smithi006.ecmyeo (1/2) 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:33:33.387 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: Rados config object exists: conf-nfs.foo 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: Creating key for client.nfs.foo.0.0.smithi006.ecmyeo-rgw 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi006.ecmyeo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: Bind address in nfs.foo.0.0.smithi006.ecmyeo's ganesha conf is defaulting to empty 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:33.388 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:33 smithi078 ceph-mon[52919]: Deploying daemon nfs.foo.0.0.smithi006.ecmyeo on smithi006 2024-05-20T22:33:35.300 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:35 smithi006 ceph-mon[65753]: pgmap v338: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 938 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-05-20T22:33:35.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:35 smithi078 ceph-mon[52919]: pgmap v338: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 938 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-05-20T22:33:36.597 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:33:37.125 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:37 smithi078 ceph-mon[52919]: pgmap v339: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 7.3 KiB/s wr, 3 op/s 2024-05-20T22:33:37.126 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:37 smithi078 ceph-mon[52919]: Evicting (and blocklisting) client session 14536 (172.21.15.6:0/1938864191) 2024-05-20T22:33:37.126 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:37 smithi078 ceph-mon[52919]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/1938864191"}]: dispatch 2024-05-20T22:33:37.126 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:37.126 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:37 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (10m) 32s ago 19m 23.6M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (7m) 32s ago 19m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (6m) 11s ago 18m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (10m) 32s ago 18m 45.3M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (39s) 32s ago 13m 12.5M - 19.0.0-2695-g8a373da3 86045c3f4e75 6b776336a4ae 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (15s) 11s ago 13m 14.5M - 19.0.0-2695-g8a373da3 86045c3f4e75 ba2434600c73 2024-05-20T22:33:37.286 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (9m) 32s ago 20m 514M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (8m) 11s ago 18m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (8m) 32s ago 20m 60.7M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (8m) 11s ago 18m 47.6M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 starting - - - - 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (12m) 11s ago 12m 88.2M - 4.0 e1d6a67b021e a193a3d9b11b 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (10m) 32s ago 18m 11.8M - 1.3.1 1dbe0e931976 b98a6a4fdfae 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (9m) 11s ago 18m 16.6M - 1.3.1 1dbe0e931976 4ff2bede2f19 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (6m) 32s ago 17m 461M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (5m) 32s ago 17m 411M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (4m) 32s ago 16m 374M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (3m) 32s ago 16m 360M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (3m) 11s ago 15m 365M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (2m) 11s ago 15m 341M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (108s) 11s ago 15m 323M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:33:37.287 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (69s) 11s ago 14m 328M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 2ec32e6186e1 2024-05-20T22:33:37.288 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (8m) 32s ago 17m 70.4M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:33:37.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:37 smithi006 ceph-mon[65753]: pgmap v339: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 7.3 KiB/s wr, 3 op/s 2024-05-20T22:33:37.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:37 smithi006 ceph-mon[65753]: Evicting (and blocklisting) client session 14536 (172.21.15.6:0/1938864191) 2024-05-20T22:33:37.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:37 smithi006 ceph-mon[65753]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/1938864191"}]: dispatch 2024-05-20T22:33:37.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:37.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:37 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:37.738 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:33:37.739 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:33:37.739 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:33:37.739 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:33:37.739 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 14 2024-05-20T22:33:37.739 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:33:37.739 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:33:38.109 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:33:38.109 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:33:38.109 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:33:38.109 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr", 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "osd", 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "mds" 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "16/23 daemons upgraded", 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading nfs daemons", 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:33:38.110 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/1938864191"}]': finished 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: osdmap e159: 8 total, 8 up, 8 in 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/431245817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:38.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:33:38.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:33:38.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:33:38.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:38.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:38.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:38.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.6:0/1938864191"}]': finished 2024-05-20T22:33:38.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: osdmap e159: 8 total, 8 up, 8 in 2024-05-20T22:33:38.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:38.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/431245817' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:38.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:38.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-05-20T22:33:38.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-05-20T22:33:38.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-05-20T22:33:38.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:38.534 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:33:39.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: pgmap v341: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 921 B/s wr, 3 op/s 2024-05-20T22:33:39.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: Upgrade: Updating nfs.foo.1.0.smithi078.metdop (2/2) 2024-05-20T22:33:39.324 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: Creating key for client.nfs.foo.1.0.smithi078.metdop 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: Rados config object exists: conf-nfs.foo 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: Creating key for client.nfs.foo.1.0.smithi078.metdop-rgw 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: Bind address in nfs.foo.1.0.smithi078.metdop's ganesha conf is defaulting to empty 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: Deploying daemon nfs.foo.1.0.smithi078.metdop on smithi078 2024-05-20T22:33:39.325 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:39 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3598451740' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:33:39.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: pgmap v341: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 921 B/s wr, 3 op/s 2024-05-20T22:33:39.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: Upgrade: Updating nfs.foo.1.0.smithi078.metdop (2/2) 2024-05-20T22:33:39.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: Creating key for client.nfs.foo.1.0.smithi078.metdop 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: from='client.34556 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: Rados config object exists: conf-nfs.foo 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: Creating key for client.nfs.foo.1.0.smithi078.metdop-rgw 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi078.metdop-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: Bind address in nfs.foo.1.0.smithi078.metdop's ganesha conf is defaulting to empty 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: Deploying daemon nfs.foo.1.0.smithi078.metdop on smithi078 2024-05-20T22:33:39.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:39 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3598451740' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:33:41.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:41 smithi078 ceph-mon[52919]: pgmap v342: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 921 B/s wr, 3 op/s 2024-05-20T22:33:41.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:41 smithi006 ceph-mon[65753]: pgmap v342: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 921 B/s wr, 3 op/s 2024-05-20T22:33:43.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:43 smithi006 ceph-mon[65753]: pgmap v343: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:43.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:43.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:43.477 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:43 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:43 smithi078 ceph-mon[52919]: pgmap v343: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:43.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:43 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:45.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:45 smithi006 ceph-mon[65753]: pgmap v344: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-05-20T22:33:45.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:45.638 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:45 smithi078 ceph-mon[52919]: pgmap v344: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-05-20T22:33:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:45.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:46.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:46 smithi006 ceph-mon[65753]: Evicting (and blocklisting) client session 24375 (172.21.15.78:0/2587796291) 2024-05-20T22:33:46.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:46 smithi006 ceph-mon[65753]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/2587796291"}]: dispatch 2024-05-20T22:33:46.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:46.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:46 smithi078 ceph-mon[52919]: Evicting (and blocklisting) client session 24375 (172.21.15.78:0/2587796291) 2024-05-20T22:33:46.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:46 smithi078 ceph-mon[52919]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/2587796291"}]: dispatch 2024-05-20T22:33:46.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:33:47.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:47 smithi078 ceph-mon[52919]: pgmap v345: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:47.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:47 smithi078 ceph-mon[52919]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/2587796291"}]': finished 2024-05-20T22:33:47.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:47 smithi078 ceph-mon[52919]: osdmap e160: 8 total, 8 up, 8 in 2024-05-20T22:33:47.348 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:47.349 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:47 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:47.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:47 smithi006 ceph-mon[65753]: pgmap v345: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.1 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:47.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:47 smithi006 ceph-mon[65753]: from='mds.? [v2:172.21.15.6:6834/3025345418,v1:172.21.15.6:6835/3025345418]' entity='mds.foofs.smithi006.pburlb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.78:0/2587796291"}]': finished 2024-05-20T22:33:47.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:47 smithi006 ceph-mon[65753]: osdmap e160: 8 total, 8 up, 8 in 2024-05-20T22:33:47.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:47.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:47 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:48.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:48 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:48.698 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:48 smithi078 ceph-mon[52919]: pgmap v347: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-05-20T22:33:48.699 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:48 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:49.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:48 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:49.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:48 smithi006 ceph-mon[65753]: pgmap v347: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-05-20T22:33:49.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:48 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:49.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:49 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:49.895 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:49 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:50.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:49 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:50.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:49 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:51.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:51 smithi006 ceph-mon[65753]: pgmap v348: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:51.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:51 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:51.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:51 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:51.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:51 smithi078 ceph-mon[52919]: pgmap v348: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:51.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:51 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:51.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:51 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: pgmap v349: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:53.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:53 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: pgmap v349: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:53.536 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:53 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:54.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:54.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all nfs 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.ecmyeo"}]: dispatch 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.ecmyeo"}]': finished 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi078.metdop"}]: dispatch 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi078.metdop"}]': finished 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:33:54.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:54 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:54.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:54.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:54.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all nfs 2024-05-20T22:33:54.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:54.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.ecmyeo"}]: dispatch 2024-05-20T22:33:54.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi006.ecmyeo"}]': finished 2024-05-20T22:33:54.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi078.metdop"}]: dispatch 2024-05-20T22:33:54.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi078.metdop"}]': finished 2024-05-20T22:33:54.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:33:54.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: Upgrade: Setting container_image for all nvmeof 2024-05-20T22:33:54.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:54 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:33:55.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:55 smithi078 ceph-mon[52919]: pgmap v350: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 3 op/s 2024-05-20T22:33:55.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:55 smithi006 ceph-mon[65753]: pgmap v350: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 3 op/s 2024-05-20T22:33:56.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:56 smithi078 ceph-mon[52919]: Upgrade: Updating node-exporter.smithi006 (1/2) 2024-05-20T22:33:56.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:56 smithi078 ceph-mon[52919]: Deploying daemon node-exporter.smithi006 on smithi006 2024-05-20T22:33:56.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:56 smithi006 ceph-mon[65753]: Upgrade: Updating node-exporter.smithi006 (1/2) 2024-05-20T22:33:56.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:56 smithi006 ceph-mon[65753]: Deploying daemon node-exporter.smithi006 on smithi006 2024-05-20T22:33:57.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:57 smithi006 ceph-mon[65753]: pgmap v351: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1023 B/s wr, 3 op/s 2024-05-20T22:33:57.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:57 smithi078 ceph-mon[52919]: pgmap v351: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 1023 B/s wr, 3 op/s 2024-05-20T22:33:59.388 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:33:59 smithi006 ceph-mon[65753]: pgmap v352: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 907 B/s wr, 3 op/s 2024-05-20T22:33:59.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:33:59 smithi078 ceph-mon[52919]: pgmap v352: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 KiB/s rd, 907 B/s wr, 3 op/s 2024-05-20T22:34:01.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:01 smithi078 ceph-mon[52919]: pgmap v353: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:34:01.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:01.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:01.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:01.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:01 smithi006 ceph-mon[65753]: pgmap v353: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:34:01.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:01.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:01.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:03.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:03 smithi078 ceph-mon[52919]: pgmap v354: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:34:03.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:03 smithi078 ceph-mon[52919]: Upgrade: Updating node-exporter.smithi078 (2/2) 2024-05-20T22:34:03.316 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:03 smithi078 ceph-mon[52919]: Deploying daemon node-exporter.smithi078 on smithi078 2024-05-20T22:34:03.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:03 smithi006 ceph-mon[65753]: pgmap v354: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-05-20T22:34:03.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:03 smithi006 ceph-mon[65753]: Upgrade: Updating node-exporter.smithi078 (2/2) 2024-05-20T22:34:03.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:03 smithi006 ceph-mon[65753]: Deploying daemon node-exporter.smithi078 on smithi078 2024-05-20T22:34:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:05 smithi078 ceph-mon[52919]: pgmap v355: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:05.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:05 smithi006 ceph-mon[65753]: pgmap v355: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:07 smithi078 ceph-mon[52919]: pgmap v356: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:07 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:07 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:07 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:07 smithi006 ceph-mon[65753]: pgmap v356: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:07 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:07 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:07 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:08.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:08 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:08.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:08 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:08.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:08 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:08.563 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:08 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:08.919 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:34:09.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:09 smithi006 ceph-mon[65753]: pgmap v357: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (10m) 22s ago 19m 23.6M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (7m) 22s ago 19m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (7m) 24s ago 19m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (10m) 22s ago 19m 53.5M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (72s) 22s ago 13m 33.6M - 19.0.0-2695-g8a373da3 86045c3f4e75 6b776336a4ae 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (47s) 24s ago 13m 14.6M - 19.0.0-2695-g8a373da3 86045c3f4e75 ba2434600c73 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (10m) 22s ago 21m 516M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:34:09.625 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (9m) 24s ago 18m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (9m) 22s ago 21m 62.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (8m) 24s ago 18m 48.1M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (33s) 22s ago 13m 55.2M - 5.7 86045c3f4e75 ac50f778762f 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (27s) 24s ago 13m 17.1M - 5.7 86045c3f4e75 d9b7900e3493 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 starting - - - - 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 starting - - - - 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (6m) 22s ago 18m 460M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (6m) 22s ago 17m 412M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (5m) 22s ago 17m 377M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (4m) 22s ago 16m 361M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (3m) 24s ago 16m 365M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:34:09.626 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (2m) 24s ago 16m 340M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:34:09.627 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (2m) 24s ago 15m 324M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:34:09.627 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (102s) 24s ago 15m 328M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 2ec32e6186e1 2024-05-20T22:34:09.627 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (9m) 22s ago 18m 70.4M - 2.33.4 514e6a882f6e e7d1812bf567 2024-05-20T22:34:09.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:09 smithi078 ceph-mon[52919]: pgmap v357: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:10.055 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:34:10.055 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:34:10.055 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:34:10.055 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:10.055 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:34:10.055 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 14 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:34:10.056 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:34:10.441 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:34:10.441 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:34:10.441 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:34:10.441 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:34:10.441 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr", 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "nfs", 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "osd", 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "mds" 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "18/23 daemons upgraded", 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading node-exporter daemons", 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:34:10.442 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:34:10.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:10 smithi078 ceph-mon[52919]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:10.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:10 smithi078 ceph-mon[52919]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:10.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:10.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:10 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:10.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:10 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3817692069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:10.686 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:10 smithi006 ceph-mon[65753]: from='client.34582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:10.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:10 smithi006 ceph-mon[65753]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:10.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:10.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:10 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:10.687 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:10 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3817692069' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:10.868 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:34:11.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:11 smithi006 ceph-mon[65753]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:11.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:11 smithi006 ceph-mon[65753]: pgmap v358: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:11.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:11 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/232586555' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:34:11.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:11.535 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:11 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:11.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:11 smithi078 ceph-mon[52919]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:11.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:11 smithi078 ceph-mon[52919]: pgmap v358: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:11.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:11 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/232586555' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:34:11.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:11.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:11 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:12.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:12 smithi078 ceph-mon[52919]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:12.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:12 smithi006 ceph-mon[65753]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:13.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:13 smithi078 ceph-mon[52919]: pgmap v359: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:13.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:13.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:13 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:13.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:13 smithi006 ceph-mon[65753]: pgmap v359: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:13.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:13.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:13 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:14 smithi078 ceph-mon[52919]: pgmap v360: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:14 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:15.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:14 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:15.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:14 smithi006 ceph-mon[65753]: pgmap v360: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:15.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:14 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:15.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:14 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:16.188 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:16.188 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:16.188 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:15 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:16.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:16.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:16.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:15 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:17.243 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:16 smithi006 ceph-mon[65753]: pgmap v361: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:17.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:16 smithi078 ceph-mon[52919]: pgmap v361: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: pgmap v362: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.680 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:18.681 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: pgmap v362: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:19.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:19.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:34:19.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:34:19.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:19.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:19.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:19.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:19.148 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:21.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:20 smithi006 ceph-mon[65753]: Upgrade: Updating prometheus.smithi006 2024-05-20T22:34:21.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:20 smithi006 ceph-mon[65753]: pgmap v363: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:21.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:20 smithi006 ceph-mon[65753]: Deploying daemon prometheus.smithi006 on smithi006 2024-05-20T22:34:21.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:20 smithi078 ceph-mon[52919]: Upgrade: Updating prometheus.smithi006 2024-05-20T22:34:21.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:20 smithi078 ceph-mon[52919]: pgmap v363: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:21.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:20 smithi078 ceph-mon[52919]: Deploying daemon prometheus.smithi006 on smithi006 2024-05-20T22:34:23.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:22 smithi006 ceph-mon[65753]: pgmap v364: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:23.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:22 smithi078 ceph-mon[52919]: pgmap v364: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:25.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:24 smithi006 ceph-mon[65753]: pgmap v365: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:25.144 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:24 smithi078 ceph-mon[52919]: pgmap v365: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:26.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:26 smithi006 ceph-mon[65753]: pgmap v366: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:27.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:26 smithi078 ceph-mon[52919]: pgmap v366: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:29.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:28 smithi006 ceph-mon[65753]: pgmap v367: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:29.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:28 smithi078 ceph-mon[52919]: pgmap v367: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:30.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:30 smithi006 ceph-mon[65753]: pgmap v368: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:30.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:31.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:30 smithi078 ceph-mon[52919]: pgmap v368: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:31.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:33.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:33 smithi006 ceph-mon[65753]: pgmap v369: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:33.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:33.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:33.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:33 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:33.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:33 smithi078 ceph-mon[52919]: pgmap v369: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:33.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:33.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:33.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:33 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:35.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:35 smithi078 ceph-mon[52919]: pgmap v370: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:35.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:35 smithi006 ceph-mon[65753]: pgmap v370: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:37.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:37 smithi006 ceph-mon[65753]: pgmap v371: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:37.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:37 smithi078 ceph-mon[52919]: pgmap v371: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:39.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:38 smithi006 ceph-mon[65753]: pgmap v372: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:39.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:39.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:38 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:39.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:38 smithi078 ceph-mon[52919]: pgmap v372: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:39.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:39.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:38 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:40.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:40.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:39 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:40.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:40.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:39 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:41.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:40 smithi006 ceph-mon[65753]: pgmap v373: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:41.060 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:40 smithi078 ceph-mon[52919]: pgmap v373: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:41.240 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:34:41.943 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:34:41.943 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (11m) 4s ago 20m 23.6M - 0.23.0 ba2b418f427c ced7816679ce 2024-05-20T22:34:41.943 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (8m) 4s ago 20m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (8m) 31s ago 19m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (11m) 4s ago 19m 53.9M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (104s) 4s ago 14m 34.1M - 19.0.0-2695-g8a373da3 86045c3f4e75 6b776336a4ae 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (79s) 31s ago 14m 14.7M - 19.0.0-2695-g8a373da3 86045c3f4e75 ba2434600c73 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (10m) 4s ago 21m 519M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (9m) 31s ago 19m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (9m) 4s ago 21m 62.5M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (9m) 31s ago 19m 49.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (66s) 4s ago 14m 56.0M - 5.7 86045c3f4e75 ac50f778762f 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (60s) 31s ago 13m 61.0M - 5.7 86045c3f4e75 d9b7900e3493 2024-05-20T22:34:41.944 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (42s) 4s ago 19m 8586k - 1.5.0 0da6a335fe13 538b0c0726e9 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (35s) 31s ago 19m 3355k - 1.5.0 0da6a335fe13 4fd410968a7c 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (7m) 4s ago 18m 461M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (6m) 4s ago 18m 413M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (5m) 4s ago 17m 379M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (4m) 4s ago 17m 362M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (4m) 31s ago 17m 366M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (3m) 31s ago 16m 341M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (2m) 31s ago 16m 325M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (2m) 31s ago 15m 329M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 2ec32e6186e1 2024-05-20T22:34:41.945 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (10s) 4s ago 19m 45.8M - 2.43.0 a07b618ecd1d 7785948673e9 2024-05-20T22:34:42.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:41 smithi078 ceph-mon[52919]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:42.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:41 smithi006 ceph-mon[65753]: from='client.34606 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:42.363 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 14 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:34:42.364 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:34:42.732 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:34:42.732 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:34:42.732 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:34:42.732 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:34:42.732 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:34:42.732 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr", 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "nfs", 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "osd", 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "mds" 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "18/23 daemons upgraded", 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading alertmanager daemons", 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:34:42.733 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:34:43.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: pgmap v374: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3041614679' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.036 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.037 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.037 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.037 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.037 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:42 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='client.34610 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: pgmap v374: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='client.34614 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3041614679' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:34:43.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.146 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.147 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:42 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:34:43.164 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:34:44.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:43 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:34:44.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:43 smithi078 ceph-mon[52919]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:44.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:43 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/4162741567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:34:44.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:43 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-05-20T22:34:44.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:43 smithi006 ceph-mon[65753]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:34:44.257 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:43 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/4162741567' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:34:45.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:44 smithi078 ceph-mon[52919]: pgmap v375: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:45.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:44 smithi078 ceph-mon[52919]: Upgrade: Updating alertmanager.smithi006 2024-05-20T22:34:45.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:44 smithi078 ceph-mon[52919]: Deploying daemon alertmanager.smithi006 on smithi006 2024-05-20T22:34:45.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:44 smithi006 ceph-mon[65753]: pgmap v375: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:45.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:44 smithi006 ceph-mon[65753]: Upgrade: Updating alertmanager.smithi006 2024-05-20T22:34:45.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:44 smithi006 ceph-mon[65753]: Deploying daemon alertmanager.smithi006 on smithi006 2024-05-20T22:34:46.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:45 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:46.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:45 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:34:47.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:46 smithi078 ceph-mon[52919]: pgmap v376: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:47.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:46 smithi006 ceph-mon[65753]: pgmap v376: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:49.128 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:48 smithi006 ceph-mon[65753]: pgmap v377: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:49.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:48 smithi078 ceph-mon[52919]: pgmap v377: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:51.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:50 smithi006 ceph-mon[65753]: pgmap v378: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:51.144 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:50 smithi078 ceph-mon[52919]: pgmap v378: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:53.135 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:52 smithi006 ceph-mon[65753]: pgmap v379: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:53.144 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:52 smithi078 ceph-mon[52919]: pgmap v379: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:55.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:55 smithi006 ceph-mon[65753]: pgmap v380: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:55.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:55.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:55.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:55 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:55.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:55 smithi078 ceph-mon[52919]: pgmap v380: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:55.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:55.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:34:55.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:55 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:34:57.394 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:57 smithi006 ceph-mon[65753]: pgmap v381: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:57.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:57 smithi078 ceph-mon[52919]: pgmap v381: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:34:59.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:34:59 smithi078 ceph-mon[52919]: pgmap v382: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:34:59.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:34:59 smithi006 ceph-mon[65753]: pgmap v382: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:00.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:00.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:00 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:00.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:00.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:00 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:01.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:01 smithi006 ceph-mon[65753]: pgmap v383: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:01.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:01.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:01.346 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:01.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:01 smithi078 ceph-mon[52919]: pgmap v383: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:01.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:01.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:01.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:03.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:03 smithi006 ceph-mon[65753]: pgmap v384: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:03.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:03 smithi078 ceph-mon[52919]: pgmap v384: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: pgmap v385: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:35:05.138 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.139 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.140 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:05 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: pgmap v385: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:05.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:05 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:07 smithi078 ceph-mon[52919]: pgmap v386: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:07 smithi078 ceph-mon[52919]: Upgrade: Updating grafana.smithi006 2024-05-20T22:35:07.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:07 smithi078 ceph-mon[52919]: Deploying daemon grafana.smithi006 on smithi006 2024-05-20T22:35:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:07 smithi006 ceph-mon[65753]: pgmap v386: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:07 smithi006 ceph-mon[65753]: Upgrade: Updating grafana.smithi006 2024-05-20T22:35:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:07 smithi006 ceph-mon[65753]: Deploying daemon grafana.smithi006 on smithi006 2024-05-20T22:35:09.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:09 smithi078 ceph-mon[52919]: pgmap v387: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:09.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:09 smithi006 ceph-mon[65753]: pgmap v387: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:11.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:11 smithi078 ceph-mon[52919]: pgmap v388: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:11.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:11 smithi006 ceph-mon[65753]: pgmap v388: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:13.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:13 smithi078 ceph-mon[52919]: pgmap v389: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:13.468 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:13 smithi006 ceph-mon[65753]: pgmap v389: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:13.537 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:35:14.227 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (20s) 14s ago 20m 16.1M - 0.25.0 c8568f914cd2 fed2550efc44 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (8m) 14s ago 20m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (8m) 64s ago 20m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (11m) 14s ago 20m 53.9M - 8.3.5 dad864ee21e9 1f95afa80b13 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (2m) 14s ago 14m 34.2M - 19.0.0-2695-g8a373da3 86045c3f4e75 6b776336a4ae 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (112s) 64s ago 14m 14.7M - 19.0.0-2695-g8a373da3 86045c3f4e75 ba2434600c73 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (11m) 14s ago 22m 519M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (10m) 64s ago 20m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (10m) 14s ago 22m 63.2M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (9m) 64s ago 19m 49.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:35:14.228 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (98s) 14s ago 14m 64.5M - 5.7 86045c3f4e75 ac50f778762f 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (92s) 64s ago 14m 61.0M - 5.7 86045c3f4e75 d9b7900e3493 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (74s) 14s ago 20m 8274k - 1.5.0 0da6a335fe13 538b0c0726e9 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (68s) 64s ago 19m 3355k - 1.5.0 0da6a335fe13 4fd410968a7c 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (7m) 14s ago 19m 462M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (7m) 14s ago 18m 413M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (6m) 14s ago 18m 377M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (5m) 14s ago 17m 363M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (4m) 64s ago 17m 366M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (4m) 64s ago 17m 341M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (3m) 64s ago 16m 325M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (2m) 64s ago 16m 329M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 2ec32e6186e1 2024-05-20T22:35:14.229 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (42s) 14s ago 19m 50.7M - 2.43.0 a07b618ecd1d 7785948673e9 2024-05-20T22:35:14.650 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:35:14.650 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 14 2024-05-20T22:35:14.651 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:35:14.652 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:35:15.001 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:35:15.001 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": true, 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "Upgrading all daemon types on all hosts", 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [ 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr", 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "nfs", 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "osd", 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "crash", 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "mon", 2024-05-20T22:35:15.002 INFO:teuthology.orchestra.run.smithi006.stdout: "mds" 2024-05-20T22:35:15.003 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:35:15.003 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": "18/23 daemons upgraded", 2024-05-20T22:35:15.003 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "Currently upgrading grafana daemons", 2024-05-20T22:35:15.003 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:35:15.003 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:35:15.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:15 smithi078 ceph-mon[52919]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:15.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:15 smithi078 ceph-mon[52919]: pgmap v390: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:15.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:15 smithi078 ceph-mon[52919]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:15.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:15 smithi078 ceph-mon[52919]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:15.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:15 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/74852865' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:15.438 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:35:15.438 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:15 smithi006 ceph-mon[65753]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:15.438 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:15 smithi006 ceph-mon[65753]: pgmap v390: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:15.438 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:15 smithi006 ceph-mon[65753]: from='client.34634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:15.438 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:15 smithi006 ceph-mon[65753]: from='client.34638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:15.439 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:15 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/74852865' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:16.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:16 smithi006 ceph-mon[65753]: from='client.34646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:16.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:16 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1256035766' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:35:16.324 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:16.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:16 smithi078 ceph-mon[52919]: from='client.34646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:16.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:16 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1256035766' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:35:16.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:17.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:17 smithi078 ceph-mon[52919]: pgmap v391: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:17.412 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:17 smithi006 ceph-mon[65753]: pgmap v391: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:19.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:18 smithi006 ceph-mon[65753]: pgmap v392: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:19.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:19.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:19.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:19.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:19.035 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:18 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:19.144 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:18 smithi078 ceph-mon[52919]: pgmap v392: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:19.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:18 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:21.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:20 smithi006 ceph-mon[65753]: pgmap v393: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:21.144 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:20 smithi078 ceph-mon[52919]: pgmap v393: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:23.034 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:22 smithi006 ceph-mon[65753]: pgmap v394: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:23.144 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:22 smithi078 ceph-mon[52919]: pgmap v394: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:24.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:24 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:24.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:24 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:24.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:24 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:24.784 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:24 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:25.340 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:25 smithi006 ceph-mon[65753]: pgmap v395: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:25.340 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:25 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:25.340 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:25 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:25.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:25 smithi078 ceph-mon[52919]: pgmap v395: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:25.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:25 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:25.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:25 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:27.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:27 smithi006 ceph-mon[65753]: pgmap v396: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:27.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:27 smithi078 ceph-mon[52919]: pgmap v396: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:29.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: pgmap v397: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:29.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.285 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: Upgrade: Finalizing container_image settings 2024-05-20T22:35:29.286 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-20T22:35:29.287 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:35:29.288 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: Upgrade: Complete! 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:29.289 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:29.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:29.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:29.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:29.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.290 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:28 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: pgmap v397: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:29.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.396 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: Upgrade: Finalizing container_image settings 2024-05-20T22:35:29.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-05-20T22:35:29.398 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.399 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: Upgrade: Complete! 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:29.400 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:28 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:31.268 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:30 smithi006 ceph-mon[65753]: pgmap v398: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:31.269 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:31.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:30 smithi078 ceph-mon[52919]: pgmap v398: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:31.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:31.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:35:33.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:32 smithi006 ceph-mon[65753]: pgmap v399: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:33.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:32 smithi078 ceph-mon[52919]: pgmap v399: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:35.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:34 smithi006 ceph-mon[65753]: pgmap v400: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:35.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:34 smithi078 ceph-mon[52919]: pgmap v400: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:37.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:36 smithi006 ceph-mon[65753]: pgmap v401: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:37.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:36 smithi078 ceph-mon[52919]: pgmap v401: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:39.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:38 smithi006 ceph-mon[65753]: pgmap v402: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:39.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:38 smithi078 ceph-mon[52919]: pgmap v402: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:41.266 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:40 smithi006 ceph-mon[65753]: pgmap v403: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:41.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:40 smithi078 ceph-mon[52919]: pgmap v403: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:43.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:42 smithi006 ceph-mon[65753]: pgmap v404: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:43.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:42 smithi078 ceph-mon[52919]: pgmap v404: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:45.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:44 smithi006 ceph-mon[65753]: pgmap v405: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:45.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:44 smithi078 ceph-mon[52919]: pgmap v405: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:46.100 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:46.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:35:46.402 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:35:47.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:47 smithi006 ceph-mon[65753]: pgmap v406: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:47.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:47 smithi006 ceph-mon[65753]: from='client.34654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:47.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:47 smithi078 ceph-mon[52919]: pgmap v406: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:47.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:47 smithi078 ceph-mon[52919]: from='client.34654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (54s) 25s ago 21m 16.1M - 0.25.0 c8568f914cd2 fed2550efc44 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (9m) 25s ago 21m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (9m) 98s ago 20m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (31s) 25s ago 21m 53.2M - 9.4.12 2bacad6d85d8 4bfb775b58d5 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (2m) 25s ago 15m 34.4M - 19.0.0-2695-g8a373da3 86045c3f4e75 6b776336a4ae 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (2m) 98s ago 15m 14.7M - 19.0.0-2695-g8a373da3 86045c3f4e75 ba2434600c73 2024-05-20T22:35:48.403 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (11m) 25s ago 22m 526M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (11m) 98s ago 20m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (10m) 25s ago 22m 64.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (10m) 98s ago 20m 49.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (2m) 25s ago 15m 71.4M - 5.7 86045c3f4e75 ac50f778762f 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (2m) 98s ago 15m 61.0M - 5.7 86045c3f4e75 d9b7900e3493 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (108s) 25s ago 21m 8590k - 1.5.0 0da6a335fe13 538b0c0726e9 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (102s) 98s ago 20m 3355k - 1.5.0 0da6a335fe13 4fd410968a7c 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (8m) 25s ago 19m 463M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (7m) 25s ago 19m 414M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (6m) 25s ago 18m 378M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (6m) 25s ago 18m 363M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (5m) 98s ago 18m 366M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (4m) 98s ago 17m 341M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:35:48.404 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (3m) 98s ago 17m 325M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:35:48.405 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (3m) 98s ago 16m 329M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 2ec32e6186e1 2024-05-20T22:35:48.405 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (76s) 25s ago 20m 51.0M - 2.43.0 a07b618ecd1d 7785948673e9 2024-05-20T22:35:48.999 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T22:35:49.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:49 smithi006 ceph-mon[65753]: pgmap v407: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:49.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:49 smithi078 ceph-mon[52919]: pgmap v407: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:50.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:50 smithi006 ceph-mon[65753]: from='client.34658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:50.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:50 smithi078 ceph-mon[52919]: from='client.34658 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:35:51.084 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:35:51.085 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:35:51.086 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:35:51.086 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 14 2024-05-20T22:35:51.086 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:35:51.086 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:35:51.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:51 smithi006 ceph-mon[65753]: pgmap v408: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:51.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:51 smithi078 ceph-mon[52919]: pgmap v408: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:51.753 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-05-20T22:35:52.145 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:52 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2724612405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:52.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:52 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2724612405' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:35:53.029 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:53 smithi006 ceph-mon[65753]: pgmap v409: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:53.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:53 smithi078 ceph-mon[52919]: pgmap v409: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:53.508 INFO:teuthology.orchestra.run.smithi006.stdout:wait for servicemap items w/ changing names to refresh 2024-05-20T22:35:54.093 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'sleep 60' 2024-05-20T22:35:55.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:55 smithi006 ceph-mon[65753]: pgmap v410: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:55.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:55 smithi078 ceph-mon[52919]: pgmap v410: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:57.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:57 smithi006 ceph-mon[65753]: pgmap v411: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:57.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:57 smithi078 ceph-mon[52919]: pgmap v411: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:35:59.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:35:59 smithi006 ceph-mon[65753]: pgmap v412: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:35:59.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:35:59 smithi078 ceph-mon[52919]: pgmap v412: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:01.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:01 smithi006 ceph-mon[65753]: pgmap v413: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:01.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:01.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:01 smithi078 ceph-mon[52919]: pgmap v413: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:01.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:03.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:03 smithi006 ceph-mon[65753]: pgmap v414: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:03.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:03 smithi078 ceph-mon[52919]: pgmap v414: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:05.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:05 smithi006 ceph-mon[65753]: pgmap v415: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:05.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:05 smithi078 ceph-mon[52919]: pgmap v415: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:07.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:07 smithi078 ceph-mon[52919]: pgmap v416: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:07.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:07 smithi006 ceph-mon[65753]: pgmap v416: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:09.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:09 smithi078 ceph-mon[52919]: pgmap v417: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:09.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:09 smithi006 ceph-mon[65753]: pgmap v417: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:11.397 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:11 smithi078 ceph-mon[52919]: pgmap v418: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:11.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:11 smithi006 ceph-mon[65753]: pgmap v418: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:13.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:13 smithi078 ceph-mon[52919]: pgmap v419: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:13.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:13 smithi006 ceph-mon[65753]: pgmap v419: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:15.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:15 smithi078 ceph-mon[52919]: pgmap v420: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:15.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:15 smithi006 ceph-mon[65753]: pgmap v420: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:16.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:16.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:17.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:17 smithi078 ceph-mon[52919]: pgmap v421: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:17.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:17 smithi006 ceph-mon[65753]: pgmap v421: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:19.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:19 smithi078 ceph-mon[52919]: pgmap v422: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:19.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:19 smithi006 ceph-mon[65753]: pgmap v422: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:21.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:21 smithi078 ceph-mon[52919]: pgmap v423: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:21.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:21 smithi006 ceph-mon[65753]: pgmap v423: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:23.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:23 smithi078 ceph-mon[52919]: pgmap v424: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:23.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:23 smithi006 ceph-mon[65753]: pgmap v424: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:25.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:25 smithi078 ceph-mon[52919]: pgmap v425: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:25.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:25 smithi006 ceph-mon[65753]: pgmap v425: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:27.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:27 smithi078 ceph-mon[52919]: pgmap v426: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:27.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:27 smithi006 ceph-mon[65753]: pgmap v426: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:29.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:29 smithi078 ceph-mon[52919]: pgmap v427: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:29.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:29 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:29.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:29 smithi006 ceph-mon[65753]: pgmap v427: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:29.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:29 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-05-20T22:36:30.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:36:30.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:36:30.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:36:30.395 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:30 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:36:30.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-05-20T22:36:30.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-05-20T22:36:30.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:36:30.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:30 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' 2024-05-20T22:36:31.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:31 smithi006 ceph-mon[65753]: pgmap v428: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:31.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:31 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:31.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:31 smithi078 ceph-mon[52919]: pgmap v428: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:31.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:31 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:33.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:33 smithi006 ceph-mon[65753]: pgmap v429: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:33.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:33 smithi078 ceph-mon[52919]: pgmap v429: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:35.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:35 smithi006 ceph-mon[65753]: pgmap v430: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:35.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:35 smithi078 ceph-mon[52919]: pgmap v430: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:37.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:37 smithi006 ceph-mon[65753]: pgmap v431: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:37.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:37 smithi078 ceph-mon[52919]: pgmap v431: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:39.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:39 smithi006 ceph-mon[65753]: pgmap v432: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:39.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:39 smithi078 ceph-mon[52919]: pgmap v432: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:41.447 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:41 smithi078 ceph-mon[52919]: pgmap v433: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:41.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:41 smithi006 ceph-mon[65753]: pgmap v433: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:43.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:43 smithi006 ceph-mon[65753]: pgmap v434: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:43.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:43 smithi078 ceph-mon[52919]: pgmap v434: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:45.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:45 smithi006 ceph-mon[65753]: pgmap v435: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:45.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:45 smithi078 ceph-mon[52919]: pgmap v435: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:46.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:46 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:46.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:46 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:36:47.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:47 smithi006 ceph-mon[65753]: pgmap v436: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:47.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:47 smithi078 ceph-mon[52919]: pgmap v436: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:49.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:49 smithi006 ceph-mon[65753]: pgmap v437: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:49.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:49 smithi078 ceph-mon[52919]: pgmap v437: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:51.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:51 smithi006 ceph-mon[65753]: pgmap v438: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:51.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:51 smithi078 ceph-mon[52919]: pgmap v438: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:53.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:53 smithi006 ceph-mon[65753]: pgmap v439: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:53.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:53 smithi078 ceph-mon[52919]: pgmap v439: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:55.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:55 smithi006 ceph-mon[65753]: pgmap v440: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:55.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:55 smithi078 ceph-mon[52919]: pgmap v440: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:56.369 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch ps' 2024-05-20T22:36:57.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:57 smithi006 ceph-mon[65753]: pgmap v441: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:57.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:57 smithi078 ceph-mon[52919]: pgmap v441: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:36:58.368 INFO:teuthology.orchestra.run.smithi006.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:alertmanager.smithi006 smithi006 *:9093,9094 running (2m) 95s ago 22m 16.1M - 0.25.0 c8568f914cd2 fed2550efc44 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi006 smithi006 running (10m) 95s ago 22m 6665k - 19.0.0-2695-g8a373da3 86045c3f4e75 57d01bccc320 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:crash.smithi078 smithi078 running (10m) 2m ago 21m 6669k - 19.0.0-2695-g8a373da3 86045c3f4e75 c9ba6379f73e 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:grafana.smithi006 smithi006 *:3000 running (101s) 95s ago 22m 53.2M - 9.4.12 2bacad6d85d8 4bfb775b58d5 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi006.pburlb smithi006 running (4m) 95s ago 16m 34.4M - 19.0.0-2695-g8a373da3 86045c3f4e75 6b776336a4ae 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:mds.foofs.smithi078.yocghn smithi078 running (3m) 2m ago 16m 14.7M - 19.0.0-2695-g8a373da3 86045c3f4e75 ba2434600c73 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi006.ssyogp smithi006 *:8443,9283,8765 running (13m) 95s ago 23m 526M - 19.0.0-2695-g8a373da3 86045c3f4e75 195e367fe079 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:mgr.smithi078.vxzcbt smithi078 *:8443,9283,8765 running (12m) 2m ago 21m 454M - 19.0.0-2695-g8a373da3 86045c3f4e75 ff58078c2ae5 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi006 smithi006 running (11m) 95s ago 24m 64.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 7db33fd7bf1f 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:mon.smithi078 smithi078 running (11m) 2m ago 21m 49.3M 2048M 19.0.0-2695-g8a373da3 86045c3f4e75 ec5bf42c857c 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.0.0.smithi006.ecmyeo smithi006 *:2049 running (3m) 95s ago 16m 71.4M - 5.7 86045c3f4e75 ac50f778762f 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:nfs.foo.1.0.smithi078.metdop smithi078 *:2049 running (3m) 2m ago 16m 61.0M - 5.7 86045c3f4e75 d9b7900e3493 2024-05-20T22:36:58.369 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi006 smithi006 *:9100 running (2m) 95s ago 22m 8590k - 1.5.0 0da6a335fe13 538b0c0726e9 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:node-exporter.smithi078 smithi078 *:9100 running (2m) 2m ago 21m 3355k - 1.5.0 0da6a335fe13 4fd410968a7c 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.0 smithi006 running (9m) 95s ago 21m 463M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 4f65f81f560e 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.1 smithi006 running (8m) 95s ago 20m 414M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 e235fe14466d 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.2 smithi006 running (8m) 95s ago 20m 378M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 2b05adf2b057 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.3 smithi006 running (7m) 95s ago 19m 363M 1937M 19.0.0-2695-g8a373da3 86045c3f4e75 b90d07d0f5f4 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.4 smithi078 running (6m) 2m ago 19m 366M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 c332350301f2 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.5 smithi078 running (5m) 2m ago 18m 341M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 ff01021e2b38 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.6 smithi078 running (5m) 2m ago 18m 325M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 811c56af46dd 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:osd.7 smithi078 running (4m) 2m ago 18m 329M 2704M 19.0.0-2695-g8a373da3 86045c3f4e75 2ec32e6186e1 2024-05-20T22:36:58.370 INFO:teuthology.orchestra.run.smithi006.stdout:prometheus.smithi006 smithi006 *:9095 running (2m) 95s ago 21m 51.0M - 2.43.0 a07b618ecd1d 7785948673e9 2024-05-20T22:36:59.002 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph orch upgrade status' 2024-05-20T22:36:59.283 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:59 smithi006 ceph-mon[65753]: pgmap v442: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:59.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:36:59 smithi006 ceph-mon[65753]: from='client.34666 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:36:59.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:59 smithi078 ceph-mon[52919]: pgmap v442: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:36:59.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:36:59 smithi078 ceph-mon[52919]: from='client.34666 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:37:00.984 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:37:00.984 INFO:teuthology.orchestra.run.smithi006.stdout: "target_image": null, 2024-05-20T22:37:00.984 INFO:teuthology.orchestra.run.smithi006.stdout: "in_progress": false, 2024-05-20T22:37:00.984 INFO:teuthology.orchestra.run.smithi006.stdout: "which": "", 2024-05-20T22:37:00.984 INFO:teuthology.orchestra.run.smithi006.stdout: "services_complete": [], 2024-05-20T22:37:00.984 INFO:teuthology.orchestra.run.smithi006.stdout: "progress": null, 2024-05-20T22:37:00.985 INFO:teuthology.orchestra.run.smithi006.stdout: "message": "", 2024-05-20T22:37:00.985 INFO:teuthology.orchestra.run.smithi006.stdout: "is_paused": false 2024-05-20T22:37:00.985 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:37:01.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:01 smithi006 ceph-mon[65753]: pgmap v443: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:01.267 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:01 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:37:01.623 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph health detail' 2024-05-20T22:37:01.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:01 smithi078 ceph-mon[52919]: pgmap v443: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:01.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:01 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:37:02.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:02 smithi006 ceph-mon[65753]: from='client.34670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:37:02.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:02 smithi078 ceph-mon[52919]: from='client.34670 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:37:03.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:03 smithi006 ceph-mon[65753]: pgmap v444: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:03.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:03 smithi078 ceph-mon[52919]: pgmap v444: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:03.663 INFO:teuthology.orchestra.run.smithi006.stdout:HEALTH_OK 2024-05-20T22:37:04.252 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions' 2024-05-20T22:37:04.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:04 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3570985553' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:37:04.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:04 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3570985553' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-05-20T22:37:05.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:05 smithi006 ceph-mon[65753]: pgmap v445: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:05.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:05 smithi078 ceph-mon[52919]: pgmap v445: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout:{ 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: "mon": { 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: "mgr": { 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: "osd": { 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 8 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:37:06.271 INFO:teuthology.orchestra.run.smithi006.stdout: "mds": { 2024-05-20T22:37:06.272 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 2 2024-05-20T22:37:06.272 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:37:06.272 INFO:teuthology.orchestra.run.smithi006.stdout: "overall": { 2024-05-20T22:37:06.272 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)": 14 2024-05-20T22:37:06.272 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:37:06.272 INFO:teuthology.orchestra.run.smithi006.stdout:} 2024-05-20T22:37:06.848 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-05-20T22:37:07.492 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:07 smithi006 ceph-mon[65753]: pgmap v446: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:07.493 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:07 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3695251029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:37:07.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:07 smithi078 ceph-mon[52919]: pgmap v446: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:07.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:07 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3695251029' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:37:08.998 INFO:teuthology.orchestra.run.smithi006.stdout:true 2024-05-20T22:37:09.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:09 smithi006 ceph-mon[65753]: pgmap v447: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:37:09.284 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:09 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/3026004612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:37:09.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:09 smithi078 ceph-mon[52919]: pgmap v447: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:37:09.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:09 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/3026004612' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:37:09.649 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -e sha1=8a373da3c102b503bf0a94126977cebf498afc8b -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-05-20T22:37:11.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:11 smithi006 ceph-mon[65753]: pgmap v448: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:11.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:11 smithi078 ceph-mon[52919]: pgmap v448: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:11.665 INFO:teuthology.orchestra.run.smithi006.stdout: "ceph version 19.0.0-2695-g8a373da3 (8a373da3c102b503bf0a94126977cebf498afc8b) squid (dev)" 2024-05-20T22:37:12.266 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-05-20T22:37:12.266 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-05-20T22:37:12.267 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- ceph orch ls -f json 2024-05-20T22:37:12.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:12 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/2815117907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:37:12.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:12 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/2815117907' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-05-20T22:37:13.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:13 smithi006 ceph-mon[65753]: pgmap v449: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:13.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:13 smithi078 ceph-mon[52919]: pgmap v449: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:14.322 INFO:teuthology.orchestra.run.smithi006.stdout: 2024-05-20T22:37:14.323 INFO:teuthology.orchestra.run.smithi006.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-05-20T22:13:47.162244Z", "last_refresh": "2024-05-20T22:35:23.287094Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-05-20T22:13:40.965658Z", "last_refresh": "2024-05-20T22:34:10.000765Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-05-20T22:13:43.760647Z", "last_refresh": "2024-05-20T22:35:23.287271Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-05-20T22:20:16.380327Z", "last_refresh": "2024-05-20T22:34:10.002080Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-05-20T22:13:39.883751Z", "last_refresh": "2024-05-20T22:34:10.000955Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi006:172.21.15.6=smithi006", "smithi078:172.21.15.78=smithi078"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-05-20T22:14:30.115230Z", "last_refresh": "2024-05-20T22:34:10.001117Z", "running": 2, "size": 2}}, {"events": ["2024-05-20T22:36:29.173699Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-05-20T22:20:36.349660Z", "last_refresh": "2024-05-20T22:34:10.002238Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-05-20T22:13:45.827224Z", "last_refresh": "2024-05-20T22:34:10.001276Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72", "last_refresh": "2024-05-20T22:34:10.001445Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-05-20T22:13:42.659589Z", "last_refresh": "2024-05-20T22:35:23.287447Z", "ports": [9095], "running": 1, "size": 1}}] 2024-05-20T22:37:15.033 INFO:tasks.cephadm:nfs.foo has 2/2 2024-05-20T22:37:15.033 DEBUG:teuthology.parallel:result is None 2024-05-20T22:37:15.034 INFO:teuthology.run_tasks:Running task vip.exec... 2024-05-20T22:37:15.044 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:37:15.044 DEBUG:teuthology.orchestra.run.smithi006:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-05-20T22:37:15.073 INFO:teuthology.orchestra.run.smithi006.stderr:+ umount /mnt/foo 2024-05-20T22:37:15.086 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T22:37:15.128 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:37:15.129 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-05-20T22:37:15.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:15 smithi006 ceph-mon[65753]: pgmap v450: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:15.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:15 smithi006 ceph-mon[65753]: from='client.34690 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:37:15.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:15 smithi078 ceph-mon[52919]: pgmap v450: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:15.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:15 smithi078 ceph-mon[52919]: from='client.34690 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-05-20T22:37:16.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:16 smithi006 ceph-mon[65753]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:37:16.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:16 smithi078 ceph-mon[52919]: from='mgr.34104 172.21.15.6:0/4037714930' entity='mgr.smithi006.ssyogp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-05-20T22:37:17.227 INFO:teuthology.orchestra.run.smithi006.stdout: "foo" 2024-05-20T22:37:17.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:17 smithi006 ceph-mon[65753]: pgmap v451: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:17.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:17 smithi078 ceph-mon[52919]: pgmap v451: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:17.892 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-05-20T22:37:18.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:18 smithi006 ceph-mon[65753]: from='client.34694 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:37:18.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:18 smithi078 ceph-mon[52919]: from='client.34694 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:37:19.431 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:19 smithi006 ceph-mon[65753]: pgmap v452: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:37:19.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:19 smithi078 ceph-mon[52919]: pgmap v452: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-05-20T22:37:19.916 INFO:teuthology.orchestra.run.smithi006.stdout:[ 2024-05-20T22:37:19.916 INFO:teuthology.orchestra.run.smithi006.stdout: { 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "access_type": "RW", 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "clients": [], 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "cluster_id": "foo", 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "export_id": 1, 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "fsal": { 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "fs_name": "foofs", 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "name": "CEPH", 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "user_id": "nfs.foo.1" 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: }, 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "path": "/", 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "protocols": [ 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: 4 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: ], 2024-05-20T22:37:19.917 INFO:teuthology.orchestra.run.smithi006.stdout: "pseudo": "/fake", 2024-05-20T22:37:19.918 INFO:teuthology.orchestra.run.smithi006.stdout: "security_label": true, 2024-05-20T22:37:19.918 INFO:teuthology.orchestra.run.smithi006.stdout: "squash": "none", 2024-05-20T22:37:19.918 INFO:teuthology.orchestra.run.smithi006.stdout: "transports": [ 2024-05-20T22:37:19.918 INFO:teuthology.orchestra.run.smithi006.stdout: "TCP" 2024-05-20T22:37:19.918 INFO:teuthology.orchestra.run.smithi006.stdout: ] 2024-05-20T22:37:19.918 INFO:teuthology.orchestra.run.smithi006.stdout: } 2024-05-20T22:37:19.918 INFO:teuthology.orchestra.run.smithi006.stdout:] 2024-05-20T22:37:20.516 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-05-20T22:37:21.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:21 smithi006 ceph-mon[65753]: pgmap v453: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:21.533 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:21 smithi006 ceph-mon[65753]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:37:21.645 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:21 smithi078 ceph-mon[52919]: pgmap v453: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:21.646 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:21 smithi078 ceph-mon[52919]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-05-20T22:37:22.334 INFO:teuthology.orchestra.run.smithi006.stdout:foo rec-0000000000000004:nfs.foo.1 2024-05-20T22:37:22.334 INFO:teuthology.orchestra.run.smithi006.stdout:foo grace 2024-05-20T22:37:22.334 INFO:teuthology.orchestra.run.smithi006.stdout:foo rec-0000000000000004:nfs.foo.0 2024-05-20T22:37:22.334 INFO:teuthology.orchestra.run.smithi006.stdout:foo conf-nfs.foo 2024-05-20T22:37:22.334 INFO:teuthology.orchestra.run.smithi006.stdout:foo export-1 2024-05-20T22:37:23.052 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-05-20T22:37:23.061 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:37:23.062 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 -- bash -c 'set -ex 2024-05-20T22:37:23.062 DEBUG:teuthology.orchestra.run.smithi006:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-05-20T22:37:23.062 DEBUG:teuthology.orchestra.run.smithi006:> ' 2024-05-20T22:37:23.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:23 smithi006 ceph-mon[65753]: pgmap v454: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:23.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:23 smithi006 ceph-mon[65753]: mgrmap e43: smithi006.ssyogp(active, since 12m), standbys: smithi078.vxzcbt 2024-05-20T22:37:23.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:23 smithi078 ceph-mon[52919]: pgmap v454: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:23.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:23 smithi078 ceph-mon[52919]: mgrmap e43: smithi006.ssyogp(active, since 12m), standbys: smithi078.vxzcbt 2024-05-20T22:37:24.815 INFO:teuthology.orchestra.run.smithi006.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-05-20T22:37:25.055 INFO:teuthology.orchestra.run.smithi006.stderr:+ [[ 6 -gt 2 ]] 2024-05-20T22:37:25.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:25 smithi006 ceph-mon[65753]: pgmap v455: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:25.369 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:25 smithi006 ceph-mon[65753]: from='client.? 172.21.15.6:0/1069013596' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-05-20T22:37:25.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:25 smithi078 ceph-mon[52919]: pgmap v455: 97 pgs: 97 active+clean; 237 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-05-20T22:37:25.644 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:25 smithi078 ceph-mon[52919]: from='client.? 172.21.15.6:0/1069013596' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-05-20T22:37:25.679 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-05-20T22:37:25.689 INFO:tasks.cephadm:Teardown begin 2024-05-20T22:37:25.689 DEBUG:teuthology.orchestra.run.smithi006:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:37:25.721 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:37:25.752 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-05-20T22:37:25.753 DEBUG:teuthology.orchestra.run.smithi006:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-20T22:37:25.778 DEBUG:teuthology.orchestra.run.smithi078:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-05-20T22:37:25.808 INFO:tasks.cephadm:Stopping all daemons... 2024-05-20T22:37:25.809 INFO:tasks.cephadm.mon.smithi006:Stopping mon.smithi006... 2024-05-20T22:37:25.809 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006 2024-05-20T22:37:26.033 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:25 smithi006 systemd[1]: Stopping Ceph mon.smithi006 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:37:26.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:26 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[65749]: 2024-05-20T22:37:26.145+0000 7f982aa3d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi006 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:37:26.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:26 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006[65749]: 2024-05-20T22:37:26.145+0000 7f982aa3d700 -1 mon.smithi006@0(leader) e3 *** Got Signal Terminated *** 2024-05-20T22:37:26.534 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:26 smithi006 podman[110801]: 2024-05-20 22:37:26.360731623 +0000 UTC m=+0.342489766 container died 7db33fd7bf1f0a4b5a63a458e17a2825bd89f094559ec049fc9a0d7093911db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-05-20T22:37:26.999 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:26 smithi006 podman[110801]: 2024-05-20 22:37:26.770770677 +0000 UTC m=+0.752528798 container cleanup 7db33fd7bf1f0a4b5a63a458e17a2825bd89f094559ec049fc9a0d7093911db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True) 2024-05-20T22:37:26.999 INFO:journalctl@ceph.mon.smithi006.smithi006.stdout:May 20 22:37:26 smithi006 bash[110801]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi006 2024-05-20T22:37:27.149 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi006.service' 2024-05-20T22:37:27.195 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:37:27.195 INFO:tasks.cephadm.mon.smithi006:Stopped mon.smithi006 2024-05-20T22:37:27.195 INFO:tasks.cephadm.mon.smithi078:Stopping mon.smithi078... 2024-05-20T22:37:27.196 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi078 2024-05-20T22:37:27.617 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:27 smithi078 systemd[1]: Stopping Ceph mon.smithi078 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:37:27.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:27 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078[52915]: 2024-05-20T22:37:27.622+0000 7f6cb6313700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi078 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:37:27.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:27 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078[52915]: 2024-05-20T22:37:27.622+0000 7f6cb6313700 -1 mon.smithi078@1(peon) e3 *** Got Signal Terminated *** 2024-05-20T22:37:27.894 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:27 smithi078 podman[78295]: 2024-05-20 22:37:27.837266652 +0000 UTC m=+0.437864829 container died ec5bf42c857c6a58e5efe5c4bb784cf7469e3b762ef84584c36ecfa6fa56d6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:37:28.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:28 smithi078 podman[78295]: 2024-05-20 22:37:28.289081399 +0000 UTC m=+0.889679571 container cleanup ec5bf42c857c6a58e5efe5c4bb784cf7469e3b762ef84584c36ecfa6fa56d6ac (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:37:28.394 INFO:journalctl@ceph.mon.smithi078.smithi078.stdout:May 20 22:37:28 smithi078 bash[78295]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-mon-smithi078 2024-05-20T22:37:28.765 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@mon.smithi078.service' 2024-05-20T22:37:28.811 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:37:28.811 INFO:tasks.cephadm.mon.smithi078:Stopped mon.smithi078 2024-05-20T22:37:28.811 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-05-20T22:37:28.811 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.0 2024-05-20T22:37:29.111 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:28 smithi006 systemd[1]: Stopping Ceph osd.0 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:37:29.533 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:29 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[71890]: 2024-05-20T22:37:29.116+0000 7f521c3b3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:37:29.534 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:29 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[71890]: 2024-05-20T22:37:29.116+0000 7f521c3b3700 -1 osd.0 160 *** Got signal Terminated *** 2024-05-20T22:37:29.534 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:29 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0[71890]: 2024-05-20T22:37:29.116+0000 7f521c3b3700 -1 osd.0 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:37:34.887 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:34 smithi006 podman[110933]: 2024-05-20 22:37:34.594257674 +0000 UTC m=+5.578628402 container died 4f65f81f560e80ca7c366b1937165cb07b661ecc5506cb94fbae39386bc9a315 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux ) 2024-05-20T22:37:35.237 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:34 smithi006 podman[110933]: 2024-05-20 22:37:34.975394567 +0000 UTC m=+5.959765295 container cleanup 4f65f81f560e80ca7c366b1937165cb07b661ecc5506cb94fbae39386bc9a315 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514) 2024-05-20T22:37:35.237 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:34 smithi006 bash[110933]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0 2024-05-20T22:37:35.533 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:35 smithi006 podman[110982]: 2024-05-20 22:37:35.243647767 +0000 UTC m=+0.648460408 container remove 4f65f81f560e80ca7c366b1937165cb07b661ecc5506cb94fbae39386bc9a315 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True) 2024-05-20T22:37:35.933 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:35 smithi006 podman[111072]: 2024-05-20 22:37:35.514336563 +0000 UTC m=+0.018596196 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:37:35.933 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:35 smithi006 podman[111072]: 2024-05-20 22:37:35.611308448 +0000 UTC m=+0.115568068 container create bb19fee5c0a0017d5f76a294178b26a68ba99da5fc78fd6201a5064835d82004 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-05-20T22:37:35.933 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:35 smithi006 podman[111072]: 2024-05-20 22:37:35.855762705 +0000 UTC m=+0.360022324 container init bb19fee5c0a0017d5f76a294178b26a68ba99da5fc78fd6201a5064835d82004 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-05-20T22:37:35.934 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:35 smithi006 podman[111072]: 2024-05-20 22:37:35.859398071 +0000 UTC m=+0.363657693 container start bb19fee5c0a0017d5f76a294178b26a68ba99da5fc78fd6201a5064835d82004 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS) 2024-05-20T22:37:35.934 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:35 smithi006 podman[111072]: 2024-05-20 22:37:35.93924421 +0000 UTC m=+0.443503840 container attach bb19fee5c0a0017d5f76a294178b26a68ba99da5fc78fd6201a5064835d82004 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240514, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:37:36.283 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:36 smithi006 podman[111072]: 2024-05-20 22:37:36.16845295 +0000 UTC m=+0.672712578 container died bb19fee5c0a0017d5f76a294178b26a68ba99da5fc78fd6201a5064835d82004 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:37:36.705 INFO:journalctl@ceph.osd.0.smithi006.stdout:May 20 22:37:36 smithi006 podman[111098]: 2024-05-20 22:37:36.709836251 +0000 UTC m=+0.534168013 container remove bb19fee5c0a0017d5f76a294178b26a68ba99da5fc78fd6201a5064835d82004 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-05-20T22:37:36.722 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.0.service' 2024-05-20T22:37:36.764 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:37:36.764 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-05-20T22:37:36.764 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-05-20T22:37:36.764 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.1 2024-05-20T22:37:37.033 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:36 smithi006 systemd[1]: Stopping Ceph osd.1 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:37:37.533 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:37 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[74480]: 2024-05-20T22:37:37.129+0000 7f46e9cfd700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:37:37.534 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:37 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[74480]: 2024-05-20T22:37:37.129+0000 7f46e9cfd700 -1 osd.1 160 *** Got signal Terminated *** 2024-05-20T22:37:37.534 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:37 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1[74480]: 2024-05-20T22:37:37.129+0000 7f46e9cfd700 -1 osd.1 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:37:42.926 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:42 smithi006 podman[111177]: 2024-05-20 22:37:42.641474652 +0000 UTC m=+5.639928145 container died e235fe14466d0bc0d43c1b373792e48653c07fd4d45e584e8b40efa4860ea08a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514) 2024-05-20T22:37:43.283 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:43 smithi006 podman[111177]: 2024-05-20 22:37:43.12176707 +0000 UTC m=+6.120220608 container cleanup e235fe14466d0bc0d43c1b373792e48653c07fd4d45e584e8b40efa4860ea08a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-05-20T22:37:43.283 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:43 smithi006 bash[111177]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1 2024-05-20T22:37:43.739 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:43 smithi006 podman[111195]: 2024-05-20 22:37:43.39014217 +0000 UTC m=+0.746567977 container remove e235fe14466d0bc0d43c1b373792e48653c07fd4d45e584e8b40efa4860ea08a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:37:44.033 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:43 smithi006 podman[111286]: 2024-05-20 22:37:43.649053121 +0000 UTC m=+0.018039165 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:37:44.034 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:43 smithi006 podman[111286]: 2024-05-20 22:37:43.766058596 +0000 UTC m=+0.135044619 container create f3b02ee8e33840617750e8ff3ecbf4d5d60af39ac7fdbbc0801ea7565a43ca36 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-05-20T22:37:44.034 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:43 smithi006 podman[111286]: 2024-05-20 22:37:43.985702158 +0000 UTC m=+0.354688197 container init f3b02ee8e33840617750e8ff3ecbf4d5d60af39ac7fdbbc0801ea7565a43ca36 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS) 2024-05-20T22:37:44.034 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:43 smithi006 podman[111286]: 2024-05-20 22:37:43.989884933 +0000 UTC m=+0.358870973 container start f3b02ee8e33840617750e8ff3ecbf4d5d60af39ac7fdbbc0801ea7565a43ca36 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=) 2024-05-20T22:37:44.534 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:44 smithi006 podman[111286]: 2024-05-20 22:37:44.093883803 +0000 UTC m=+0.462869828 container attach f3b02ee8e33840617750e8ff3ecbf4d5d60af39ac7fdbbc0801ea7565a43ca36 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:37:44.534 INFO:journalctl@ceph.osd.1.smithi006.stdout:May 20 22:37:44 smithi006 podman[111286]: 2024-05-20 22:37:44.297680423 +0000 UTC m=+0.666666447 container died f3b02ee8e33840617750e8ff3ecbf4d5d60af39ac7fdbbc0801ea7565a43ca36 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-1-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-05-20T22:37:44.880 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.1.service' 2024-05-20T22:37:44.924 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:37:44.924 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-05-20T22:37:44.924 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-05-20T22:37:44.924 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.2 2024-05-20T22:37:45.271 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:45 smithi006 systemd[1]: Stopping Ceph osd.2 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:37:45.534 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:45 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[77179]: 2024-05-20T22:37:45.276+0000 7f90865fe700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:37:45.534 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:45 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[77179]: 2024-05-20T22:37:45.276+0000 7f90865fe700 -1 osd.2 160 *** Got signal Terminated *** 2024-05-20T22:37:45.534 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:45 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2[77179]: 2024-05-20T22:37:45.276+0000 7f90865fe700 -1 osd.2 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:37:51.211 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:50 smithi006 podman[111393]: 2024-05-20 22:37:50.871252247 +0000 UTC m=+5.705139314 container died 2b05adf2b057f702fe7ea9ece3444e4b0619cdc9dfc85e7858007292c698950d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-05-20T22:37:51.488 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:51 smithi006 podman[111393]: 2024-05-20 22:37:51.217738973 +0000 UTC m=+6.051626047 container cleanup 2b05adf2b057f702fe7ea9ece3444e4b0619cdc9dfc85e7858007292c698950d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-05-20T22:37:51.488 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:51 smithi006 bash[111393]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2 2024-05-20T22:37:51.783 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:51 smithi006 podman[111410]: 2024-05-20 22:37:51.494629442 +0000 UTC m=+0.625093521 container remove 2b05adf2b057f702fe7ea9ece3444e4b0619cdc9dfc85e7858007292c698950d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:37:52.183 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:51 smithi006 podman[111539]: 2024-05-20 22:37:51.79371008 +0000 UTC m=+0.021170715 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:37:52.184 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:51 smithi006 podman[111539]: 2024-05-20 22:37:51.895632154 +0000 UTC m=+0.123092758 container create bb20e5ce71dabde933bdd61f4b8e0a9256f5c3dca4c10f316f59b4271f0e6633 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2) 2024-05-20T22:37:52.184 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:52 smithi006 podman[111539]: 2024-05-20 22:37:52.098078839 +0000 UTC m=+0.325539450 container init bb20e5ce71dabde933bdd61f4b8e0a9256f5c3dca4c10f316f59b4271f0e6633 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2) 2024-05-20T22:37:52.184 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:52 smithi006 podman[111539]: 2024-05-20 22:37:52.101528713 +0000 UTC m=+0.328989316 container start bb20e5ce71dabde933bdd61f4b8e0a9256f5c3dca4c10f316f59b4271f0e6633 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:37:52.184 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:52 smithi006 podman[111539]: 2024-05-20 22:37:52.189865809 +0000 UTC m=+0.417326427 container attach bb20e5ce71dabde933bdd61f4b8e0a9256f5c3dca4c10f316f59b4271f0e6633 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-05-20T22:37:52.533 INFO:journalctl@ceph.osd.2.smithi006.stdout:May 20 22:37:52 smithi006 podman[111539]: 2024-05-20 22:37:52.430557853 +0000 UTC m=+0.658018462 container died bb20e5ce71dabde933bdd61f4b8e0a9256f5c3dca4c10f316f59b4271f0e6633 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514) 2024-05-20T22:37:53.086 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.2.service' 2024-05-20T22:37:53.134 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:37:53.137 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-05-20T22:37:53.137 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-05-20T22:37:53.138 DEBUG:teuthology.orchestra.run.smithi006:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.3 2024-05-20T22:37:53.458 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:53 smithi006 systemd[1]: Stopping Ceph osd.3 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:37:53.783 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:53 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[79795]: 2024-05-20T22:37:53.463+0000 7f697fb64700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:37:53.783 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:53 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[79795]: 2024-05-20T22:37:53.463+0000 7f697fb64700 -1 osd.3 160 *** Got signal Terminated *** 2024-05-20T22:37:53.784 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:53 smithi006 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3[79795]: 2024-05-20T22:37:53.463+0000 7f697fb64700 -1 osd.3 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:37:55.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:37:55 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:37:55.284+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:37:55.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:37:55 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:37:55.380+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:37:56.644 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:37:56 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:37:56.176+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:37:56.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:37:56 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:37:56.245+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:37:56.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:37:56 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:37:56.336+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:37:57.614 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:37:57 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:37:57.207+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:37:57.614 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:37:57 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:37:57.367+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:37:57.615 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:37:57 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:37:57.281+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:37:57.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:37:57 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:37:57.619+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:37:58.615 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:37:58 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:37:58.267+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:37:58.616 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:37:58 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:37:58.343+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:37:58.616 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:37:58 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:37:58.192+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:37:58.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:37:58 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:37:58.621+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:37:59.183 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:58 smithi006 podman[111647]: 2024-05-20 22:37:58.913952219 +0000 UTC m=+5.565011257 container died b90d07d0f5f4dfc6023f13bd7c8acc251d1a61ce6becc6b680847eb57a5f674f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:37:59.533 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:59 smithi006 podman[111647]: 2024-05-20 22:37:59.363843849 +0000 UTC m=+6.014902879 container cleanup b90d07d0f5f4dfc6023f13bd7c8acc251d1a61ce6becc6b680847eb57a5f674f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:37:59.534 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:59 smithi006 bash[111647]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3 2024-05-20T22:37:59.586 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:37:59 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:37:59.199+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:37:59.586 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:37:59 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:37:59.218+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:37:59.586 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:37:59 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:37:59.362+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:37:59.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:37:59 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:37:59.591+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:37:59.991 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:59 smithi006 podman[111664]: 2024-05-20 22:37:59.559702467 +0000 UTC m=+0.639308370 container remove b90d07d0f5f4dfc6023f13bd7c8acc251d1a61ce6becc6b680847eb57a5f674f (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-20T22:38:00.283 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:59 smithi006 podman[111750]: 2024-05-20 22:37:59.901196809 +0000 UTC m=+0.018459008 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:38:00.283 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:37:59 smithi006 podman[111750]: 2024-05-20 22:37:59.999799368 +0000 UTC m=+0.117061554 container create 9eda0baef671be14b83a686b68ac679e92fe18bc0e3d3fd29c17ae790e5425a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD) 2024-05-20T22:38:00.284 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:38:00 smithi006 podman[111750]: 2024-05-20 22:38:00.21956115 +0000 UTC m=+0.336823339 container init 9eda0baef671be14b83a686b68ac679e92fe18bc0e3d3fd29c17ae790e5425a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:38:00.284 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:38:00 smithi006 podman[111750]: 2024-05-20 22:38:00.223648296 +0000 UTC m=+0.340910483 container start 9eda0baef671be14b83a686b68ac679e92fe18bc0e3d3fd29c17ae790e5425a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514) 2024-05-20T22:38:00.579 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:00 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:00.216+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:00.580 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:00 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:00.261+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:00.580 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:00 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:00.323+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:00.783 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:38:00 smithi006 podman[111750]: 2024-05-20 22:38:00.294466686 +0000 UTC m=+0.411728885 container attach 9eda0baef671be14b83a686b68ac679e92fe18bc0e3d3fd29c17ae790e5425a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:38:00.784 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:38:00 smithi006 podman[111750]: 2024-05-20 22:38:00.535786086 +0000 UTC m=+0.653048283 container died 9eda0baef671be14b83a686b68ac679e92fe18bc0e3d3fd29c17ae790e5425a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD) 2024-05-20T22:38:00.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:00 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:00.585+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:38:01.083 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:38:01 smithi006 podman[111776]: 2024-05-20 22:38:01.073037834 +0000 UTC m=+0.527058191 container remove 9eda0baef671be14b83a686b68ac679e92fe18bc0e3d3fd29c17ae790e5425a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True) 2024-05-20T22:38:01.084 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:38:01 smithi006 systemd[1]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.3.service: Deactivated successfully. 2024-05-20T22:38:01.084 INFO:journalctl@ceph.osd.3.smithi006.stdout:May 20 22:38:01 smithi006 systemd[1]: Stopped Ceph osd.3 for 00e42188-16f6-11ef-bc9a-c7b262605968. 2024-05-20T22:38:01.087 DEBUG:teuthology.orchestra.run.smithi006:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.3.service' 2024-05-20T22:38:01.127 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:38:01.127 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-05-20T22:38:01.127 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-05-20T22:38:01.127 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.4 2024-05-20T22:38:01.495 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:01 smithi078 systemd[1]: Stopping Ceph osd.4 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:38:01.496 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:01 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:01.230+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:01.496 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:01 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:01.233+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:01.496 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:01 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:01.278+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:01.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:01 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:01.501+0000 7f7d4d68e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:38:01.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:01 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:01.501+0000 7f7d4d68e700 -1 osd.4 160 *** Got signal Terminated *** 2024-05-20T22:38:01.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:01 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:01.501+0000 7f7d4d68e700 -1 osd.4 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:38:01.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:01 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:01.615+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:38:02.612 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:02 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:02.213+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:02.613 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:02 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:02.234+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:02.613 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:02 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:02.282+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:02.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:02 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:02.618+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:38:03.605 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:03.254+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:03.606 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:03.272+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:03.606 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:03.186+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:03.606 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:03.186+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:03.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:03 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:03.611+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:38:04.629 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:04 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:04.228+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:04.629 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:04 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:04.237+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:04.630 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:04 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:04.181+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:04.630 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:04 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:04.181+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:04.894 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:04 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:04.634+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:38:05.394 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:05 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:05.180+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:05.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:05 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:05.133+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:05.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:05 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:05.133+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:05.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:05 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:05.246+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:05.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:05 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:05.246+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:06.116 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:05 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:05.674+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.004223+0000 front 2024-05-20T22:37:33.004004+0000 (oldest deadline 2024-05-20T22:37:57.104048+0000) 2024-05-20T22:38:06.117 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:05 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4[57853]: 2024-05-20T22:38:05.674+0000 7f7d45c7a700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.606420+0000 front 2024-05-20T22:37:41.606440+0000 (oldest deadline 2024-05-20T22:38:05.106290+0000) 2024-05-20T22:38:06.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:06 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:06.122+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:06.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:06 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:06.122+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:06.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:06 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:06.148+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:06.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:06 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:06.148+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:06.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:06 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:06.253+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:06.396 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:06 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:06.253+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:07.136 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:06 smithi078 podman[78492]: 2024-05-20 22:38:06.983293545 +0000 UTC m=+5.652982199 container died c332350301f25a0a0f54a3ff29267131d84f6a275ae270599ef2a1c4ecbaec75 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-05-20T22:38:07.136 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:07 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:07.123+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:07.137 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:07 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:07.123+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:07.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:07 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:07.142+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:07.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:07 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:07.142+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:07.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:07 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:07.245+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:07.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:07 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:07.245+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:07.722 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:07 smithi078 podman[78492]: 2024-05-20 22:38:07.451877273 +0000 UTC m=+6.121565913 container cleanup c332350301f25a0a0f54a3ff29267131d84f6a275ae270599ef2a1c4ecbaec75 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-05-20T22:38:07.722 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:07 smithi078 bash[78492]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4 2024-05-20T22:38:08.062 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:07 smithi078 podman[78508]: 2024-05-20 22:38:07.728919119 +0000 UTC m=+0.744689422 container remove c332350301f25a0a0f54a3ff29267131d84f6a275ae270599ef2a1c4ecbaec75 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:38:08.376 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:08 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:08.148+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:08.376 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:08 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:08.148+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:08.376 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:08 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:08.145+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:08.376 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:08 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:08.145+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:08.377 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:08 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:08.196+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:08.377 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:08 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:08.196+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:08.377 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:08 smithi078 podman[78597]: 2024-05-20 22:38:07.971791936 +0000 UTC m=+0.017615583 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:38:08.377 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:08 smithi078 podman[78597]: 2024-05-20 22:38:08.071487813 +0000 UTC m=+0.117311448 container create 103dc4b97b4d5b9e29642e4eac5ebae56f0de1456a721655e73a7a64aa2fc813 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-05-20T22:38:08.377 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:08 smithi078 podman[78597]: 2024-05-20 22:38:08.299137151 +0000 UTC m=+0.344960794 container init 103dc4b97b4d5b9e29642e4eac5ebae56f0de1456a721655e73a7a64aa2fc813 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-05-20T22:38:08.377 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:08 smithi078 podman[78597]: 2024-05-20 22:38:08.303083065 +0000 UTC m=+0.348906708 container start 103dc4b97b4d5b9e29642e4eac5ebae56f0de1456a721655e73a7a64aa2fc813 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-05-20T22:38:08.378 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:08 smithi078 podman[78597]: 2024-05-20 22:38:08.38255983 +0000 UTC m=+0.428383480 container attach 103dc4b97b4d5b9e29642e4eac5ebae56f0de1456a721655e73a7a64aa2fc813 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240514, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-05-20T22:38:08.644 INFO:journalctl@ceph.osd.4.smithi078.stdout:May 20 22:38:08 smithi078 podman[78597]: 2024-05-20 22:38:08.592179804 +0000 UTC m=+0.638003459 container died 103dc4b97b4d5b9e29642e4eac5ebae56f0de1456a721655e73a7a64aa2fc813 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-05-20T22:38:09.214 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.4.service' 2024-05-20T22:38:09.252 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:38:09.253 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-05-20T22:38:09.253 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-05-20T22:38:09.253 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.5 2024-05-20T22:38:09.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:09.185+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:09.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:09.185+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:09.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:09.139+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:09.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:09.139+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:09.395 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:09.150+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:09.395 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:09.150+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:09.395 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:09 smithi078 systemd[1]: Stopping Ceph osd.5 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:38:09.894 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:09.589+0000 7f402291a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:38:09.894 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:09.589+0000 7f402291a700 -1 osd.5 160 *** Got signal Terminated *** 2024-05-20T22:38:09.894 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:09 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:09.589+0000 7f402291a700 -1 osd.5 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:38:10.395 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:10 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:10.184+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:10.395 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:10 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:10.184+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:10.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:10 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:10.105+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:10.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:10 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:10.105+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:10.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:10 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:10.208+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:10.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:10 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:10.208+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:11.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:11 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:11.138+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:11.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:11 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:11.138+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:11.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:11 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:11.122+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:11.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:11 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:11.122+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:11.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:11 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:11.224+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:11.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:11 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:11.224+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:12.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:12.135+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:12.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:12.135+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:12.394 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:12.110+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:12.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:12.110+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:12.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:12.110+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:12.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:12.194+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:12.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:12.194+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:12.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:12 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:12.194+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:13.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:13.166+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:13.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:13.166+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:13.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:13.166+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:47.586208+0000 front 2024-05-20T22:37:47.586220+0000 (oldest deadline 2024-05-20T22:38:12.885951+0000) 2024-05-20T22:38:13.394 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:13.091+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:13.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:13.091+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:13.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:13.091+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:13.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:13.242+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:13.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:13.242+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:13.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:13 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:13.242+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:14.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:14.170+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:32.984460+0000 front 2024-05-20T22:37:32.984353+0000 (oldest deadline 2024-05-20T22:37:55.284191+0000) 2024-05-20T22:38:14.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:14.170+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.385143+0000 front 2024-05-20T22:37:39.385156+0000 (oldest deadline 2024-05-20T22:38:02.885025+0000) 2024-05-20T22:38:14.394 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5[60095]: 2024-05-20T22:38:14.170+0000 7f401ef0e700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:47.586208+0000 front 2024-05-20T22:37:47.586220+0000 (oldest deadline 2024-05-20T22:38:12.885951+0000) 2024-05-20T22:38:14.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:14.126+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:14.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:14.126+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:14.395 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:14.126+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:14.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:14.292+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:14.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:14.292+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:14.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:14 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:14.292+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:15.333 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:15 smithi078 podman[78703]: 2024-05-20 22:38:15.070498156 +0000 UTC m=+5.589085676 container died ff01021e2b383c4cc7d1945999beb3b9292717198d0df1e83b0c145af64d2015 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:38:15.333 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:15 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:15.136+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:15.333 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:15 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:15.136+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:15.333 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:15 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:15.136+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:15.644 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:15 smithi078 podman[78703]: 2024-05-20 22:38:15.523523353 +0000 UTC m=+6.042110908 container cleanup ff01021e2b383c4cc7d1945999beb3b9292717198d0df1e83b0c145af64d2015 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-05-20T22:38:15.733 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:15 smithi078 bash[78703]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5 2024-05-20T22:38:15.733 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:15 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:15.339+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:15.733 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:15 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:15.339+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:15.733 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:15 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:15.339+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:16.144 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:15 smithi078 podman[78752]: 2024-05-20 22:38:15.81678566 +0000 UTC m=+0.742307531 container remove ff01021e2b383c4cc7d1945999beb3b9292717198d0df1e83b0c145af64d2015 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-05-20T22:38:16.448 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:16 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:16.315+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:16.448 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:16 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:16.315+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:16.448 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:16 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:16.315+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:16.448 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:16 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:16.181+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:16.449 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:16 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:16.181+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:16.449 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:16 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:16.181+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:16.449 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:16 smithi078 podman[78840]: 2024-05-20 22:38:16.070058648 +0000 UTC m=+0.017581902 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:38:16.449 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:16 smithi078 podman[78840]: 2024-05-20 22:38:16.22634291 +0000 UTC m=+0.173866208 container create 94d414c5bddc6ae95c9054e928b83fc870ee593c386749c29ab9ed34f099975d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-05-20T22:38:16.449 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:16 smithi078 podman[78840]: 2024-05-20 22:38:16.454988673 +0000 UTC m=+0.402511941 container init 94d414c5bddc6ae95c9054e928b83fc870ee593c386749c29ab9ed34f099975d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240514, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-05-20T22:38:16.747 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:16 smithi078 podman[78840]: 2024-05-20 22:38:16.458382076 +0000 UTC m=+0.405905329 container start 94d414c5bddc6ae95c9054e928b83fc870ee593c386749c29ab9ed34f099975d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:38:16.747 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:16 smithi078 podman[78840]: 2024-05-20 22:38:16.556322603 +0000 UTC m=+0.503845864 container attach 94d414c5bddc6ae95c9054e928b83fc870ee593c386749c29ab9ed34f099975d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-05-20T22:38:17.036 INFO:journalctl@ceph.osd.5.smithi078.stdout:May 20 22:38:16 smithi078 podman[78840]: 2024-05-20 22:38:16.754186785 +0000 UTC m=+0.701710045 container died 94d414c5bddc6ae95c9054e928b83fc870ee593c386749c29ab9ed34f099975d (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True) 2024-05-20T22:38:17.320 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:17.213+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:17.320 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:17.213+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:17.320 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:17.213+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:17.320 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:17.283+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:17.320 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:17.283+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:17.320 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:17.283+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:17.342 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.5.service' 2024-05-20T22:38:17.379 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:38:17.379 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-05-20T22:38:17.380 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-05-20T22:38:17.380 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.6 2024-05-20T22:38:17.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:17 smithi078 systemd[1]: Stopping Ceph osd.6 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:38:18.144 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:17.737+0000 7ff3466fa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:38:18.144 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:17.737+0000 7ff3466fa700 -1 osd.6 160 *** Got signal Terminated *** 2024-05-20T22:38:18.144 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:17 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:17.737+0000 7ff3466fa700 -1 osd.6 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:38:18.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:18 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:18.249+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:18.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:18 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:18.249+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:18.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:18 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:18.249+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:18.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:18 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:18.265+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:18.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:18 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:18.265+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:18.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:18 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:18.265+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:19.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:19.251+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:19.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:19.251+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:19.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:19.251+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:19.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:19.240+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:19.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:19.240+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:19.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:19.241+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:19.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:19 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:19.241+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:54.406082+0000 front 2024-05-20T22:37:54.406068+0000 (oldest deadline 2024-05-20T22:38:19.106049+0000) 2024-05-20T22:38:20.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:20.229+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:20.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:20.229+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:20.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:20.229+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:20.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:20.237+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:20.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:20.237+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:20.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:20.237+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:20.645 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:20 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:20.237+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:54.406082+0000 front 2024-05-20T22:37:54.406068+0000 (oldest deadline 2024-05-20T22:38:19.106049+0000) 2024-05-20T22:38:21.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:21.229+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:21.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:21.229+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:21.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:21.229+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:21.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:21.229+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:54.406082+0000 front 2024-05-20T22:37:54.406068+0000 (oldest deadline 2024-05-20T22:38:19.106049+0000) 2024-05-20T22:38:21.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:21.271+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:21.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:21.271+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:21.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:21 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:21.271+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:22.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:22 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:22.208+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:33.801342+0000 front 2024-05-20T22:37:33.801311+0000 (oldest deadline 2024-05-20T22:37:54.801712+0000) 2024-05-20T22:38:22.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:22 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:22.208+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:41.703491+0000 front 2024-05-20T22:37:41.703539+0000 (oldest deadline 2024-05-20T22:38:05.203676+0000) 2024-05-20T22:38:22.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:22 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:22.208+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.205273+0000 front 2024-05-20T22:37:49.205316+0000 (oldest deadline 2024-05-20T22:38:12.105118+0000) 2024-05-20T22:38:22.644 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:22 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6[62296]: 2024-05-20T22:38:22.208+0000 7ff33ece6700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:54.406082+0000 front 2024-05-20T22:37:54.406068+0000 (oldest deadline 2024-05-20T22:38:19.106049+0000) 2024-05-20T22:38:22.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:22 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:22.251+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:22.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:22 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:22.251+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:22.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:22 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:22.251+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:23.616 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:23 smithi078 podman[78945]: 2024-05-20 22:38:23.354403684 +0000 UTC m=+5.749173188 container died 811c56af46dd0a057b56eff66129caa1557c0b5d429774a0f3c86c8a38333224 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:38:23.617 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:23 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:23.222+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:23.617 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:23 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:23.222+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:23.617 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:23 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:23.222+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:23.894 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:23 smithi078 podman[78945]: 2024-05-20 22:38:23.764465109 +0000 UTC m=+6.159234602 container cleanup 811c56af46dd0a057b56eff66129caa1557c0b5d429774a0f3c86c8a38333224 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-05-20T22:38:23.894 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:23 smithi078 bash[78945]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6 2024-05-20T22:38:24.394 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:24 smithi078 podman[78962]: 2024-05-20 22:38:24.016276178 +0000 UTC m=+0.657790417 container remove 811c56af46dd0a057b56eff66129caa1557c0b5d429774a0f3c86c8a38333224 (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-05-20T22:38:24.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:24 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:24.242+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:24.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:24 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:24.243+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:24.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:24 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:24.243+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:24.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:24 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:24.243+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:58.449794+0000 front 2024-05-20T22:37:58.449809+0000 (oldest deadline 2024-05-20T22:38:23.749569+0000) 2024-05-20T22:38:24.713 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:24 smithi078 podman[79050]: 2024-05-20 22:38:24.345538404 +0000 UTC m=+0.017696189 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:38:24.713 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:24 smithi078 podman[79050]: 2024-05-20 22:38:24.475703699 +0000 UTC m=+0.147861506 container create 572a906ec16cf6f367d307f289081cb2d9b2ca2abf068a81904b934a7d8385fd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-05-20T22:38:24.713 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:24 smithi078 podman[79050]: 2024-05-20 22:38:24.72017331 +0000 UTC m=+0.392331089 container init 572a906ec16cf6f367d307f289081cb2d9b2ca2abf068a81904b934a7d8385fd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, maintainer=Guillaume Abrioux ) 2024-05-20T22:38:25.007 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:24 smithi078 podman[79050]: 2024-05-20 22:38:24.723525349 +0000 UTC m=+0.395683143 container start 572a906ec16cf6f367d307f289081cb2d9b2ca2abf068a81904b934a7d8385fd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.build-date=20240514, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-05-20T22:38:25.007 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:24 smithi078 podman[79050]: 2024-05-20 22:38:24.828410472 +0000 UTC m=+0.500568298 container attach 572a906ec16cf6f367d307f289081cb2d9b2ca2abf068a81904b934a7d8385fd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:38:25.316 INFO:journalctl@ceph.osd.6.smithi078.stdout:May 20 22:38:25 smithi078 podman[79050]: 2024-05-20 22:38:25.014282482 +0000 UTC m=+0.686440284 container died 572a906ec16cf6f367d307f289081cb2d9b2ca2abf068a81904b934a7d8385fd (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-6-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:38:25.317 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:25 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:25.222+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:25.317 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:25 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:25.222+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:25.317 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:25 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:25.222+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:25.317 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:25 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:25.222+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:58.449794+0000 front 2024-05-20T22:37:58.449809+0000 (oldest deadline 2024-05-20T22:38:23.749569+0000) 2024-05-20T22:38:25.642 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.6.service' 2024-05-20T22:38:25.681 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:38:25.681 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-05-20T22:38:25.681 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-05-20T22:38:25.682 DEBUG:teuthology.orchestra.run.smithi078:> sudo systemctl stop ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.7 2024-05-20T22:38:25.894 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:25 smithi078 systemd[1]: Stopping Ceph osd.7 for 00e42188-16f6-11ef-bc9a-c7b262605968... 2024-05-20T22:38:26.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:26.051+0000 7f2f9646d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-05-20T22:38:26.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:26.051+0000 7f2f9646d700 -1 osd.7 160 *** Got signal Terminated *** 2024-05-20T22:38:26.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:26.051+0000 7f2f9646d700 -1 osd.7 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-05-20T22:38:26.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:26.236+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:26.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:26.236+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:26.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:26.236+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:26.395 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:26 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:26.236+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:58.449794+0000 front 2024-05-20T22:37:58.449809+0000 (oldest deadline 2024-05-20T22:38:23.749569+0000) 2024-05-20T22:38:27.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:27 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:27.218+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:27.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:27 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:27.218+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:27.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:27 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:27.218+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:27.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:27 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:27.218+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:58.449794+0000 front 2024-05-20T22:37:58.449809+0000 (oldest deadline 2024-05-20T22:38:23.749569+0000) 2024-05-20T22:38:28.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:28 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:28.185+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:28.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:28 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:28.185+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:28.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:28 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:28.185+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:28.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:28 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:28.185+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:58.449794+0000 front 2024-05-20T22:37:58.449809+0000 (oldest deadline 2024-05-20T22:38:23.749569+0000) 2024-05-20T22:38:28.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:28 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:28.185+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6804 osd.4 since back 2024-05-20T22:38:03.750071+0000 front 2024-05-20T22:38:03.749973+0000 (oldest deadline 2024-05-20T22:38:27.249999+0000) 2024-05-20T22:38:29.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:29 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:29.168+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:29.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:29 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:29.168+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:29.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:29 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:29.168+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:29.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:29 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:29.168+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:58.449794+0000 front 2024-05-20T22:37:58.449809+0000 (oldest deadline 2024-05-20T22:38:23.749569+0000) 2024-05-20T22:38:29.645 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:29 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:29.168+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6804 osd.4 since back 2024-05-20T22:38:03.750071+0000 front 2024-05-20T22:38:03.749973+0000 (oldest deadline 2024-05-20T22:38:27.249999+0000) 2024-05-20T22:38:30.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:30.135+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6806 osd.0 since back 2024-05-20T22:37:31.145507+0000 front 2024-05-20T22:37:31.145318+0000 (oldest deadline 2024-05-20T22:37:55.245420+0000) 2024-05-20T22:38:30.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:30.135+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6814 osd.1 since back 2024-05-20T22:37:39.946509+0000 front 2024-05-20T22:37:39.946480+0000 (oldest deadline 2024-05-20T22:38:05.246323+0000) 2024-05-20T22:38:30.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:30.135+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6822 osd.2 since back 2024-05-20T22:37:49.248117+0000 front 2024-05-20T22:37:49.248027+0000 (oldest deadline 2024-05-20T22:38:12.048406+0000) 2024-05-20T22:38:30.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:30.135+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.6:6830 osd.3 since back 2024-05-20T22:37:58.449794+0000 front 2024-05-20T22:37:58.449809+0000 (oldest deadline 2024-05-20T22:38:23.749569+0000) 2024-05-20T22:38:30.394 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:30 smithi078 ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7[64543]: 2024-05-20T22:38:30.135+0000 7f2f8ea59700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.78:6804 osd.4 since back 2024-05-20T22:38:03.750071+0000 front 2024-05-20T22:38:03.749973+0000 (oldest deadline 2024-05-20T22:38:27.249999+0000) 2024-05-20T22:38:31.939 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:31 smithi078 podman[79166]: 2024-05-20 22:38:31.670462866 +0000 UTC m=+5.726066723 container died 2ec32e6186e13241b162df647c832f4b2a9614179722bcf144de8e7b2f86139a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-05-20T22:38:32.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:32 smithi078 podman[79166]: 2024-05-20 22:38:32.294242349 +0000 UTC m=+6.349846198 container cleanup 2ec32e6186e13241b162df647c832f4b2a9614179722bcf144de8e7b2f86139a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, org.label-schema.build-date=20240514, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-05-20T22:38:32.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:32 smithi078 bash[79166]: ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7 2024-05-20T22:38:32.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:32 smithi078 podman[79214]: 2024-05-20 22:38:32.495975242 +0000 UTC m=+0.819475212 container remove 2ec32e6186e13241b162df647c832f4b2a9614179722bcf144de8e7b2f86139a (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240514, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-05-20T22:38:33.226 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:32 smithi078 podman[79305]: 2024-05-20 22:38:32.822086247 +0000 UTC m=+0.017501856 image pull 86045c3f4e75f357c6a286e6b257c2664737befc0fde29c9c0b81b4c0f1102c0 quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72 2024-05-20T22:38:33.226 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:32 smithi078 podman[79305]: 2024-05-20 22:38:32.930285976 +0000 UTC m=+0.125701563 container create 12d3e50686bc033eaaf3a794d02c3995f3d3912a3c85c9913db01252ac4d940c (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0) 2024-05-20T22:38:33.227 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:33 smithi078 podman[79305]: 2024-05-20 22:38:33.14968841 +0000 UTC m=+0.345103996 container init 12d3e50686bc033eaaf3a794d02c3995f3d3912a3c85c9913db01252ac4d940c (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240514, io.buildah.version=1.33.5, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-05-20T22:38:33.227 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:33 smithi078 podman[79305]: 2024-05-20 22:38:33.153288723 +0000 UTC m=+0.348704319 container start 12d3e50686bc033eaaf3a794d02c3995f3d3912a3c85c9913db01252ac4d940c (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da) 2024-05-20T22:38:33.227 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:33 smithi078 podman[79305]: 2024-05-20 22:38:33.23274674 +0000 UTC m=+0.428162340 container attach 12d3e50686bc033eaaf3a794d02c3995f3d3912a3c85c9913db01252ac4d940c (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240514, maintainer=Guillaume Abrioux ) 2024-05-20T22:38:33.644 INFO:journalctl@ceph.osd.7.smithi078.stdout:May 20 22:38:33 smithi078 podman[79305]: 2024-05-20 22:38:33.449503851 +0000 UTC m=+0.644919451 container died 12d3e50686bc033eaaf3a794d02c3995f3d3912a3c85c9913db01252ac4d940c (image=quay.ceph.io/ceph-ci/ceph@sha256:42900ce9e15d5098a41b2c62f055540b50274fafdde74c804604a6d9a94f9e72, name=ceph-00e42188-16f6-11ef-bc9a-c7b262605968-osd-7-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240514, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=d4a0f93b06679a5c9148c9e9b367478b2a0d04e0, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-adk4-testing-2024-05-17-0821-squid-8a373da, ceph=True) 2024-05-20T22:38:34.029 DEBUG:teuthology.orchestra.run.smithi078:> sudo pkill -f 'journalctl -f -n 0 -u ceph-00e42188-16f6-11ef-bc9a-c7b262605968@osd.7.service' 2024-05-20T22:38:34.068 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-05-20T22:38:34.068 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-05-20T22:38:34.069 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 --force --keep-logs 2024-05-20T22:41:35.284 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 --force --keep-logs 2024-05-20T22:41:54.981 DEBUG:teuthology.orchestra.run.smithi006:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:41:55.010 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-05-20T22:41:55.038 INFO:tasks.cephadm:Archiving crash dumps... 2024-05-20T22:41:55.040 DEBUG:teuthology.misc:Transferring archived files from smithi006:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783/remote/smithi006/crash 2024-05-20T22:41:55.041 DEBUG:teuthology.orchestra.run.smithi006:> sudo tar c -f - -C /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/crash -- . 2024-05-20T22:41:55.086 INFO:teuthology.orchestra.run.smithi006.stderr:tar: /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/crash: Cannot open: No such file or directory 2024-05-20T22:41:55.086 INFO:teuthology.orchestra.run.smithi006.stderr:tar: Error is not recoverable: exiting now 2024-05-20T22:41:55.088 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/crash to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783/remote/smithi078/crash 2024-05-20T22:41:55.089 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/crash -- . 2024-05-20T22:41:55.128 INFO:teuthology.orchestra.run.smithi078.stderr:tar: /var/lib/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/crash: Cannot open: No such file or directory 2024-05-20T22:41:55.128 INFO:teuthology.orchestra.run.smithi078.stderr:tar: Error is not recoverable: exiting now 2024-05-20T22:41:55.129 INFO:tasks.cephadm:Checking cluster log for badness... 2024-05-20T22:41:55.129 DEBUG:teuthology.orchestra.run.smithi006:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-05-20T22:41:55.156 INFO:tasks.cephadm:Compressing logs... 2024-05-20T22:41:55.157 DEBUG:teuthology.orchestra.run.smithi006:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-20T22:41:55.198 DEBUG:teuthology.orchestra.run.smithi078:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-05-20T22:41:55.223 INFO:teuthology.orchestra.run.smithi006.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.log 2024-05-20T22:41:55.223 INFO:teuthology.orchestra.run.smithi006.stderr:: No such file or directory 2024-05-20T22:41:55.223 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.2.log 2024-05-20T22:41:55.223 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.audit.log 2024-05-20T22:41:55.223 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.log: /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mon.smithi006.log 2024-05-20T22:41:55.223 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.3.log 2024-05-20T22:41:55.226 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mon.smithi006.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.1.log 2024-05-20T22:41:55.226 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.0.log 2024-05-20T22:41:55.226 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.cephadm.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mon.smithi078.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:: No such file or directory 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mgr.smithi078.vxzcbt.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mon.smithi078.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mgr.smithi078.vxzcbt.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.audit.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.7.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.5.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.audit.log: /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mds.foofs.smithi078.yocghn.log 2024-05-20T22:41:55.230 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.4.log 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mds.foofs.smithi078.yocghn.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.6.log 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr: 88.6% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mds.foofs.smithi078.yocghn.log.gz 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.4.log: 87.0% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.log.gz 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.cephadm.log 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-volume.log 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.cephadm.log: 91.6% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.audit.log.gz 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-20T22:41:55.231 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-volume.log: 85.7% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.cephadm.log.gz 2024-05-20T22:41:55.233 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.0.log: 87.0% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.log.gz 2024-05-20T22:41:55.233 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mds.foofs.smithi006.pburlb.log 2024-05-20T22:41:55.233 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.cephadm.log: 85.7% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.cephadm.log.gz 2024-05-20T22:41:55.233 INFO:teuthology.orchestra.run.smithi006.stderr: 91.5% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph.audit.log.gz 2024-05-20T22:41:55.233 INFO:teuthology.orchestra.run.smithi006.stderr:gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-volume.log 2024-05-20T22:41:55.233 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mds.foofs.smithi006.pburlb.log: gzip -5 --verbose -- /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mgr.smithi006.ssyogp.log 2024-05-20T22:41:55.237 INFO:teuthology.orchestra.run.smithi078.stderr:/var/log/ceph/cephadm.log: 90.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-20T22:41:55.241 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-05-20T22:41:55.244 INFO:teuthology.orchestra.run.smithi078.stderr: 91.2% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-volume.log.gz 2024-05-20T22:41:55.247 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mgr.smithi006.ssyogp.log: 91.1% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-volume.log.gz 2024-05-20T22:41:55.254 INFO:teuthology.orchestra.run.smithi006.stderr: 91.3% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mds.foofs.smithi006.pburlb.log.gz 2024-05-20T22:41:55.274 INFO:teuthology.orchestra.run.smithi078.stderr: 89.7% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mgr.smithi078.vxzcbt.log.gz 2024-05-20T22:41:55.294 INFO:teuthology.orchestra.run.smithi006.stderr:/var/log/ceph/cephadm.log: 91.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-05-20T22:41:55.537 INFO:teuthology.orchestra.run.smithi078.stderr: 92.5% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mon.smithi078.log.gz 2024-05-20T22:41:55.601 INFO:teuthology.orchestra.run.smithi006.stderr: 88.3% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mgr.smithi006.ssyogp.log.gz 2024-05-20T22:41:57.265 INFO:teuthology.orchestra.run.smithi006.stderr: 91.9% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-mon.smithi006.log.gz 2024-05-20T22:42:08.521 INFO:teuthology.orchestra.run.smithi006.stderr: 94.4% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.2.log.gz 2024-05-20T22:42:10.335 INFO:teuthology.orchestra.run.smithi078.stderr: 93.9% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.4.log.gz 2024-05-20T22:42:12.637 INFO:teuthology.orchestra.run.smithi078.stderr: 94.3% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.6.log.gz 2024-05-20T22:42:13.085 INFO:teuthology.orchestra.run.smithi006.stderr: 94.1% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.0.log.gz 2024-05-20T22:42:14.850 INFO:teuthology.orchestra.run.smithi006.stderr: 94.2% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.1.log.gz 2024-05-20T22:42:15.342 INFO:teuthology.orchestra.run.smithi078.stderr: 94.2% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.5.log.gz 2024-05-20T22:42:15.538 INFO:teuthology.orchestra.run.smithi006.stderr: 94.1% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.3.log.gz 2024-05-20T22:42:15.540 INFO:teuthology.orchestra.run.smithi006.stderr: 2024-05-20T22:42:15.540 INFO:teuthology.orchestra.run.smithi006.stderr:real 0m20.329s 2024-05-20T22:42:15.540 INFO:teuthology.orchestra.run.smithi006.stderr:user 1m10.931s 2024-05-20T22:42:15.540 INFO:teuthology.orchestra.run.smithi006.stderr:sys 0m2.529s 2024-05-20T22:42:20.610 INFO:teuthology.orchestra.run.smithi078.stderr: 94.1% -- replaced with /var/log/ceph/00e42188-16f6-11ef-bc9a-c7b262605968/ceph-osd.7.log.gz 2024-05-20T22:42:20.612 INFO:teuthology.orchestra.run.smithi078.stderr: 2024-05-20T22:42:20.612 INFO:teuthology.orchestra.run.smithi078.stderr:real 0m25.399s 2024-05-20T22:42:20.612 INFO:teuthology.orchestra.run.smithi078.stderr:user 1m14.944s 2024-05-20T22:42:20.612 INFO:teuthology.orchestra.run.smithi078.stderr:sys 0m2.922s 2024-05-20T22:42:20.612 INFO:tasks.cephadm:Archiving logs... 2024-05-20T22:42:20.613 DEBUG:teuthology.misc:Transferring archived files from smithi006:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783/remote/smithi006/log 2024-05-20T22:42:20.614 DEBUG:teuthology.orchestra.run.smithi006:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-20T22:42:29.778 DEBUG:teuthology.misc:Transferring archived files from smithi078:/var/log/ceph to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783/remote/smithi078/log 2024-05-20T22:42:29.779 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /var/log/ceph -- . 2024-05-20T22:42:40.364 INFO:tasks.cephadm:Removing cluster... 2024-05-20T22:42:40.364 DEBUG:teuthology.orchestra.run.smithi006:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 --force 2024-05-20T22:42:40.755 DEBUG:teuthology.orchestra.run.smithi078:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 00e42188-16f6-11ef-bc9a-c7b262605968 --force 2024-05-20T22:42:41.142 INFO:tasks.cephadm:Removing cephadm ... 2024-05-20T22:42:41.143 DEBUG:teuthology.orchestra.run.smithi006:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-20T22:42:41.159 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf /home/ubuntu/cephtest/cephadm 2024-05-20T22:42:41.175 INFO:tasks.cephadm:Teardown complete 2024-05-20T22:42:41.175 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-05-20T22:42:41.186 INFO:teuthology.task.clock:Checking final clock skew... 2024-05-20T22:42:41.186 DEBUG:teuthology.orchestra.run.smithi006:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T22:42:41.203 DEBUG:teuthology.orchestra.run.smithi078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-05-20T22:42:41.215 INFO:teuthology.orchestra.run.smithi006.stderr:bash: line 1: ntpq: command not found 2024-05-20T22:42:41.220 INFO:teuthology.orchestra.run.smithi006.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T22:42:41.220 INFO:teuthology.orchestra.run.smithi006.stdout:=============================================================================== 2024-05-20T22:42:41.220 INFO:teuthology.orchestra.run.smithi006.stdout:^* hv01.front.sepia.ceph.com 3 6 377 13 +728us[ +752us] +/- 62ms 2024-05-20T22:42:41.220 INFO:teuthology.orchestra.run.smithi006.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 13 -101us[ -77us] +/- 77ms 2024-05-20T22:42:41.220 INFO:teuthology.orchestra.run.smithi006.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 14 -749us[ -725us] +/- 75ms 2024-05-20T22:42:41.220 INFO:teuthology.orchestra.run.smithi006.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:42:41.229 INFO:teuthology.orchestra.run.smithi078.stderr:bash: line 1: ntpq: command not found 2024-05-20T22:42:41.233 INFO:teuthology.orchestra.run.smithi078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-05-20T22:42:41.233 INFO:teuthology.orchestra.run.smithi078.stdout:=============================================================================== 2024-05-20T22:42:41.233 INFO:teuthology.orchestra.run.smithi078.stdout:^* hv01.front.sepia.ceph.com 3 6 377 10 +745us[ +746us] +/- 62ms 2024-05-20T22:42:41.233 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 13 -87us[ -85us] +/- 77ms 2024-05-20T22:42:41.233 INFO:teuthology.orchestra.run.smithi078.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 207 -737us[ -765us] +/- 72ms 2024-05-20T22:42:41.233 INFO:teuthology.orchestra.run.smithi078.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-05-20T22:42:41.234 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-05-20T22:42:41.245 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-05-20T22:42:41.245 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-05-20T22:42:41.257 DEBUG:teuthology.orchestra.run.smithi006:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-20T22:42:41.342 DEBUG:teuthology.orchestra.run.smithi078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-05-20T22:42:41.415 DEBUG:teuthology.orchestra.run.smithi006:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T22:42:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:42:41.444 DEBUG:teuthology.orchestra.run.smithi078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-05-20T22:42:41.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:42:41.472 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-05-20T22:42:41.482 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-05-20T22:42:41.492 INFO:teuthology.task.internal:Duration was 2096.378828 seconds 2024-05-20T22:42:41.493 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-05-20T22:42:41.504 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-05-20T22:42:41.504 DEBUG:teuthology.orchestra.run.smithi006:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-20T22:42:41.507 DEBUG:teuthology.orchestra.run.smithi078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-05-20T22:42:41.544 INFO:teuthology.orchestra.run.smithi006.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T22:42:41.550 INFO:teuthology.orchestra.run.smithi078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-05-20T22:42:42.071 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-05-20T22:42:42.071 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi006.front.sepia.ceph.com 2024-05-20T22:42:42.071 DEBUG:teuthology.orchestra.run.smithi006:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-20T22:42:42.134 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi078.front.sepia.ceph.com 2024-05-20T22:42:42.134 DEBUG:teuthology.orchestra.run.smithi078:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-05-20T22:42:42.170 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-05-20T22:42:42.170 DEBUG:teuthology.orchestra.run.smithi006:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-20T22:42:42.176 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-05-20T22:42:42.237 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-05-20T22:42:42.237 DEBUG:teuthology.orchestra.run.smithi006:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-20T22:42:42.246 DEBUG:teuthology.orchestra.run.smithi078:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-05-20T22:42:57.320 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-05-20T22:42:57.331 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-05-20T22:42:57.332 DEBUG:teuthology.orchestra.run.smithi006:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-20T22:42:57.358 DEBUG:teuthology.orchestra.run.smithi078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-05-20T22:42:57.387 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-05-20T22:42:57.398 DEBUG:teuthology.orchestra.run.smithi006:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-20T22:42:57.400 DEBUG:teuthology.orchestra.run.smithi078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-05-20T22:42:57.422 INFO:teuthology.orchestra.run.smithi006.stdout:kernel.core_pattern = core 2024-05-20T22:42:57.453 INFO:teuthology.orchestra.run.smithi078.stdout:kernel.core_pattern = core 2024-05-20T22:42:57.691 DEBUG:teuthology.orchestra.run.smithi006:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-20T22:42:57.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:42:57.707 DEBUG:teuthology.orchestra.run.smithi078:> test -e /home/ubuntu/cephtest/archive/coredump 2024-05-20T22:42:57.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-05-20T22:42:57.750 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-05-20T22:42:57.761 INFO:teuthology.task.internal:Transferring archived files... 2024-05-20T22:42:57.762 DEBUG:teuthology.misc:Transferring archived files from smithi006:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783/remote/smithi006 2024-05-20T22:42:57.762 DEBUG:teuthology.orchestra.run.smithi006:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-20T22:42:57.832 DEBUG:teuthology.misc:Transferring archived files from smithi078:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-05-20_18:23:59-orch:cephadm-wip-adk4-testing-2024-05-17-0821-squid-distro-default-smithi/7715783/remote/smithi078 2024-05-20T22:42:57.832 DEBUG:teuthology.orchestra.run.smithi078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-05-20T22:42:57.875 INFO:teuthology.task.internal:Removing archive directory... 2024-05-20T22:42:57.875 DEBUG:teuthology.orchestra.run.smithi006:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-20T22:42:57.878 DEBUG:teuthology.orchestra.run.smithi078:> rm -rf -- /home/ubuntu/cephtest/archive 2024-05-20T22:42:57.927 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-05-20T22:42:57.939 INFO:teuthology.task.internal:Not uploading archives. 2024-05-20T22:42:57.939 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-05-20T22:42:57.955 INFO:teuthology.task.internal:Tidying up after the test... 2024-05-20T22:42:57.955 DEBUG:teuthology.orchestra.run.smithi006:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-20T22:42:57.958 DEBUG:teuthology.orchestra.run.smithi078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-05-20T22:42:57.971 INFO:teuthology.orchestra.run.smithi006.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 20 22:42 /home/ubuntu/cephtest 2024-05-20T22:42:57.983 INFO:teuthology.orchestra.run.smithi078.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 May 20 22:42 /home/ubuntu/cephtest 2024-05-20T22:42:57.985 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-05-20T22:42:57.995 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-05-20T22:42:58.028 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2096.378828048706 owner: scheduled_adking@teuthology success: true 2024-05-20T22:42:58.028 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-05-20T22:42:58.104 INFO:teuthology.run:pass