2024-09-20T18:11:21.300 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-20T18:11:21.315 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-20T18:11:21.389 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659 branch: wip-guits-main-2024-09-20-1028 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7912659' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-20-1028 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 88deadb8082f4a33832e228ccefc59cedba158a5 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 88deadb8082f4a33832e228ccefc59cedba158a5 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-20-1028 sha1: 88deadb8082f4a33832e228ccefc59cedba158a5 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 9945 sha1: 88deadb8082f4a33832e228ccefc59cedba158a5 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-20-1028 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_88deadb8082f4a33832e228ccefc59cedba158a5/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 88deadb8082f4a33832e228ccefc59cedba158a5 targets: smithi012.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLMQuee1VZ79cr0a0ZXnyiUtb7vIWJ4e21SEyw4T+qRvimX0pH4lWJjT8IbOXmf+yccDGvCN0rSKuz8u21xM3d0= smithi174.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC4aaIuiREVP1pp5C4vWOU82LTJAOesu9jm+JGCxpDHsy7kNF3NLm7EBgnIX+MHJZa2kkguxfCBeOuFBnypLsfA= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-20_12:24:58 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-20T18:11:21.389 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_88deadb8082f4a33832e228ccefc59cedba158a5/qa; will attempt to use it 2024-09-20T18:11:21.390 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_88deadb8082f4a33832e228ccefc59cedba158a5/qa/tasks 2024-09-20T18:11:21.390 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-20T18:11:21.391 INFO:teuthology.task.internal:Checking packages... 2024-09-20T18:11:21.410 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '88deadb8082f4a33832e228ccefc59cedba158a5' 2024-09-20T18:11:21.410 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-20T18:11:21.410 INFO:teuthology.packaging:ref: None 2024-09-20T18:11:21.410 INFO:teuthology.packaging:tag: None 2024-09-20T18:11:21.410 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-20-1028 2024-09-20T18:11:21.410 INFO:teuthology.packaging:sha1: 88deadb8082f4a33832e228ccefc59cedba158a5 2024-09-20T18:11:21.410 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-20-1028 2024-09-20T18:11:21.551 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5099.g88deadb8 2024-09-20T18:11:21.552 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-20T18:11:21.565 INFO:teuthology.task.internal:no buildpackages task found 2024-09-20T18:11:21.565 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-20T18:11:21.602 INFO:teuthology.task.internal:Saving configuration 2024-09-20T18:11:21.615 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-20T18:11:21.631 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-20T18:11:21.656 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi012.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 18:04:28.513462', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLMQuee1VZ79cr0a0ZXnyiUtb7vIWJ4e21SEyw4T+qRvimX0pH4lWJjT8IbOXmf+yccDGvCN0rSKuz8u21xM3d0='} 2024-09-20T18:11:21.676 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi174.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-20 18:04:28.511635', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBC4aaIuiREVP1pp5C4vWOU82LTJAOesu9jm+JGCxpDHsy7kNF3NLm7EBgnIX+MHJZa2kkguxfCBeOuFBnypLsfA='} 2024-09-20T18:11:21.676 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-20T18:11:21.682 INFO:teuthology.task.internal:roles: ubuntu@smithi012.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-20T18:11:21.683 INFO:teuthology.task.internal:roles: ubuntu@smithi174.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-20T18:11:21.683 INFO:teuthology.run_tasks:Running task console_log... 2024-09-20T18:11:21.738 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f09bb7ac280>, signals=[15]) 2024-09-20T18:11:21.738 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-20T18:11:21.744 INFO:teuthology.task.internal:Opening connections... 2024-09-20T18:11:21.745 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:11:21.746 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:11:21.820 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi174.front.sepia.ceph.com 2024-09-20T18:11:21.821 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:11:21.891 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-20T18:11:21.898 DEBUG:teuthology.orchestra.run.smithi012:> uname -m 2024-09-20T18:11:21.916 INFO:teuthology.orchestra.run.smithi012.stdout:x86_64 2024-09-20T18:11:21.916 DEBUG:teuthology.orchestra.run.smithi012:> cat /etc/os-release 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:NAME="CentOS Stream" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION="9" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:ID="centos" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:ID_LIKE="rhel fedora" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:VERSION_ID="9" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:PLATFORM_ID="platform:el9" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:ANSI_COLOR="0;31" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:LOGO="fedora-logo-icon" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T18:11:21.974 INFO:teuthology.orchestra.run.smithi012.stdout:HOME_URL="https://centos.org/" 2024-09-20T18:11:21.975 INFO:teuthology.orchestra.run.smithi012.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T18:11:21.975 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T18:11:21.975 INFO:teuthology.orchestra.run.smithi012.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T18:11:21.975 INFO:teuthology.lock.ops:Updating smithi012.front.sepia.ceph.com on lock server 2024-09-20T18:11:21.999 DEBUG:teuthology.orchestra.run.smithi174:> uname -m 2024-09-20T18:11:22.017 INFO:teuthology.orchestra.run.smithi174.stdout:x86_64 2024-09-20T18:11:22.017 DEBUG:teuthology.orchestra.run.smithi174:> cat /etc/os-release 2024-09-20T18:11:22.075 INFO:teuthology.orchestra.run.smithi174.stdout:NAME="CentOS Stream" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION="9" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:ID="centos" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:ID_LIKE="rhel fedora" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:VERSION_ID="9" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:PLATFORM_ID="platform:el9" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:ANSI_COLOR="0;31" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:LOGO="fedora-logo-icon" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:HOME_URL="https://centos.org/" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-20T18:11:22.076 INFO:teuthology.orchestra.run.smithi174.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-20T18:11:22.077 INFO:teuthology.lock.ops:Updating smithi174.front.sepia.ceph.com on lock server 2024-09-20T18:11:22.099 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-20T18:11:22.107 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-20T18:11:22.113 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-20T18:11:22.113 DEBUG:teuthology.orchestra.run.smithi012:> test '!' -e /home/ubuntu/cephtest 2024-09-20T18:11:22.115 DEBUG:teuthology.orchestra.run.smithi174:> test '!' -e /home/ubuntu/cephtest 2024-09-20T18:11:22.132 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-20T18:11:22.138 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-20T18:11:22.138 DEBUG:teuthology.orchestra.run.smithi012:> test -z $(ls -A /var/lib/ceph) 2024-09-20T18:11:22.173 DEBUG:teuthology.orchestra.run.smithi174:> test -z $(ls -A /var/lib/ceph) 2024-09-20T18:11:22.206 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-20T18:11:22.258 INFO:teuthology.run_tasks:Running task kernel... 2024-09-20T18:11:22.271 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-20T18:11:22.271 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-20T18:11:22.271 DEBUG:teuthology.orchestra.run.smithi012:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T18:11:22.271 DEBUG:teuthology.orchestra.run.smithi174:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-20T18:11:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:11:22.287 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2024-09-20T18:11:22.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:11:22.288 DEBUG:teuthology.orchestra.run.smithi012:> uname -r 2024-09-20T18:11:22.343 INFO:teuthology.orchestra.run.smithi174.stdout:5.14.0-503.el9.x86_64 2024-09-20T18:11:22.343 INFO:teuthology.task.kernel:Running kernel on smithi174: 5.14.0-503.el9.x86_64 2024-09-20T18:11:22.343 DEBUG:teuthology.orchestra.run.smithi174:> sudo yum install -y kernel 2024-09-20T18:11:22.346 INFO:teuthology.orchestra.run.smithi012.stdout:5.14.0-503.el9.x86_64 2024-09-20T18:11:22.346 INFO:teuthology.task.kernel:Running kernel on smithi012: 5.14.0-503.el9.x86_64 2024-09-20T18:11:22.346 DEBUG:teuthology.orchestra.run.smithi012:> sudo yum install -y kernel 2024-09-20T18:11:24.682 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.3 MB 00:00 2024-09-20T18:11:24.702 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.3 MB 00:00 2024-09-20T18:11:26.416 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - AppStream 33 MB/s | 20 MB 00:00 2024-09-20T18:11:27.762 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - AppStream 10 MB/s | 20 MB 00:02 2024-09-20T18:11:31.100 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - CRB 8.5 MB/s | 6.5 MB 00:00 2024-09-20T18:11:32.054 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-20T18:11:33.592 INFO:teuthology.orchestra.run.smithi012.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 19 kB 00:01 2024-09-20T18:11:33.713 INFO:teuthology.orchestra.run.smithi174.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 19 kB 00:00 2024-09-20T18:11:34.554 INFO:teuthology.orchestra.run.smithi012.stdout:Extra Packages for Enterprise Linux 27 MB/s | 23 MB 00:00 2024-09-20T18:11:34.809 INFO:teuthology.orchestra.run.smithi174.stdout:Extra Packages for Enterprise Linux 23 MB/s | 23 MB 00:00 2024-09-20T18:11:40.168 INFO:teuthology.orchestra.run.smithi012.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-09-20T18:11:40.231 INFO:teuthology.orchestra.run.smithi174.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-20T18:11:42.116 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T18:11:42.116 INFO:teuthology.orchestra.run.smithi012.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T18:11:42.129 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-20T18:11:42.129 INFO:teuthology.orchestra.run.smithi174.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-20T18:11:42.211 INFO:teuthology.orchestra.run.smithi012.stdout:Dependencies resolved. 2024-09-20T18:11:42.222 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout: Package Architecture Version Repository Size 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout:Installing: 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout:Installing dependencies: 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction Summary 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout:================================================================================ 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout:Install 4 Packages 2024-09-20T18:11:42.223 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:11:42.224 INFO:teuthology.orchestra.run.smithi012.stdout:Total download size: 79 M 2024-09-20T18:11:42.224 INFO:teuthology.orchestra.run.smithi012.stdout:Installed size: 126 M 2024-09-20T18:11:42.224 INFO:teuthology.orchestra.run.smithi012.stdout:Downloading Packages: 2024-09-20T18:11:42.228 INFO:teuthology.orchestra.run.smithi174.stdout:Dependencies resolved. 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout: Package Architecture Version Repository Size 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout:Installing: 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout:Installing dependencies: 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:11:42.236 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction Summary 2024-09-20T18:11:42.237 INFO:teuthology.orchestra.run.smithi174.stdout:================================================================================ 2024-09-20T18:11:42.237 INFO:teuthology.orchestra.run.smithi174.stdout:Install 4 Packages 2024-09-20T18:11:42.237 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:11:42.237 INFO:teuthology.orchestra.run.smithi174.stdout:Total download size: 79 M 2024-09-20T18:11:42.237 INFO:teuthology.orchestra.run.smithi174.stdout:Installed size: 126 M 2024-09-20T18:11:42.237 INFO:teuthology.orchestra.run.smithi174.stdout:Downloading Packages: 2024-09-20T18:11:42.399 INFO:teuthology.orchestra.run.smithi174.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 311 kB/s | 36 kB 00:00 2024-09-20T18:11:42.824 INFO:teuthology.orchestra.run.smithi174.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 29 MB/s | 16 MB 00:00 2024-09-20T18:11:42.853 INFO:teuthology.orchestra.run.smithi012.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 62 kB/s | 36 kB 00:00 2024-09-20T18:11:43.091 INFO:teuthology.orchestra.run.smithi174.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 41 MB/s | 29 MB 00:00 2024-09-20T18:11:43.400 INFO:teuthology.orchestra.run.smithi174.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 31 MB/s | 35 MB 00:01 2024-09-20T18:11:43.400 INFO:teuthology.orchestra.run.smithi174.stdout:-------------------------------------------------------------------------------- 2024-09-20T18:11:43.400 INFO:teuthology.orchestra.run.smithi174.stdout:Total 68 MB/s | 79 MB 00:01 2024-09-20T18:11:43.670 INFO:teuthology.orchestra.run.smithi012.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-20T18:11:44.084 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction check 2024-09-20T18:11:44.353 INFO:teuthology.orchestra.run.smithi012.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-20T18:11:44.421 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction check succeeded. 2024-09-20T18:11:44.421 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction test 2024-09-20T18:11:45.003 INFO:teuthology.orchestra.run.smithi012.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 13 MB/s | 29 MB 00:02 2024-09-20T18:11:45.004 INFO:teuthology.orchestra.run.smithi012.stdout:-------------------------------------------------------------------------------- 2024-09-20T18:11:45.004 INFO:teuthology.orchestra.run.smithi012.stdout:Total 28 MB/s | 79 MB 00:02 2024-09-20T18:11:45.372 INFO:teuthology.orchestra.run.smithi174.stdout:Transaction test succeeded. 2024-09-20T18:11:45.372 INFO:teuthology.orchestra.run.smithi174.stdout:Running transaction 2024-09-20T18:11:45.716 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction check 2024-09-20T18:11:46.157 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction check succeeded. 2024-09-20T18:11:46.157 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction test 2024-09-20T18:11:47.113 INFO:teuthology.orchestra.run.smithi012.stdout:Transaction test succeeded. 2024-09-20T18:11:47.114 INFO:teuthology.orchestra.run.smithi012.stdout:Running transaction 2024-09-20T18:11:47.821 INFO:teuthology.orchestra.run.smithi174.stdout: Preparing : 1/1 2024-09-20T18:11:48.484 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T18:11:48.568 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T18:11:49.676 INFO:teuthology.orchestra.run.smithi012.stdout: Preparing : 1/1 2024-09-20T18:11:50.367 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-20T18:11:50.445 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T18:11:51.306 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T18:11:51.427 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T18:11:53.208 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T18:11:53.367 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T18:11:56.283 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T18:11:56.402 INFO:teuthology.orchestra.run.smithi174.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:11:58.487 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T18:11:58.567 INFO:teuthology.orchestra.run.smithi012.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:01.287 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:03.710 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:39.501 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:39.542 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:39.874 INFO:teuthology.orchestra.run.smithi174.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:39.874 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T18:12:39.874 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T18:12:39.874 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T18:12:40.686 INFO:teuthology.orchestra.run.smithi174.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:40.686 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:12:40.687 INFO:teuthology.orchestra.run.smithi174.stdout:Installed: 2024-09-20T18:12:40.687 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T18:12:40.687 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T18:12:40.687 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T18:12:40.687 INFO:teuthology.orchestra.run.smithi174.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T18:12:40.687 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:12:40.687 INFO:teuthology.orchestra.run.smithi174.stdout:Complete! 2024-09-20T18:12:41.058 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T18:12:41.151 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T18:12:41.151 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T18:12:41.152 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T18:12:41.152 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T18:12:41.152 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T18:12:41.152 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-20T18:12:41.152 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi174.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T18:12:41.152 DEBUG:teuthology.orchestra.run.smithi174:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T18:12:41.210 INFO:teuthology.orchestra.run.smithi174.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T18:12:41.211 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T18:12:41.211 DEBUG:teuthology.orchestra.run.smithi174:> sudo rpm -qi grub2-tools 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Name : grub2-tools 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Epoch : 1 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Version : 2.06 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Release : 82.el9 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Architecture: x86_64 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Group : Unspecified 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Size : 8274670 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:License : GPLv3+ 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T18:12:41.243 INFO:teuthology.orchestra.run.smithi174.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:Packager : builder@centos.org 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:Vendor : CentOS 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:Summary : Support tools for GRUB. 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:Description : 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:hardware devices. 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:12:41.244 INFO:teuthology.orchestra.run.smithi174.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T18:12:41.245 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T18:12:41.245 INFO:teuthology.task.kernel:Updating grub on smithi174 to boot 5.14.0-508.el9.x86_64 2024-09-20T18:12:41.245 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T18:12:41.889 INFO:teuthology.orchestra.run.smithi174.stderr:Generating grub configuration file ... 2024-09-20T18:12:42.942 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:42.983 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:43.262 INFO:teuthology.orchestra.run.smithi174.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T18:12:43.277 INFO:teuthology.orchestra.run.smithi174.stderr:done 2024-09-20T18:12:43.279 DEBUG:teuthology.orchestra.run.smithi174:> mktemp 2024-09-20T18:12:43.295 INFO:teuthology.orchestra.run.smithi174.stdout:/tmp/tmp.bM4Tl8OYB0 2024-09-20T18:12:43.295 DEBUG:teuthology.orchestra.run.smithi174:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.bM4Tl8OYB0 2024-09-20T18:12:43.367 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 0666 /tmp/tmp.bM4Tl8OYB0 2024-09-20T18:12:43.390 INFO:teuthology.orchestra.run.smithi012.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:43.390 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-20T18:12:43.390 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-20T18:12:43.390 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-20T18:12:43.508 DEBUG:teuthology.orchestra.remote:smithi174:/tmp/tmp.bM4Tl8OYB0 is 6KB 2024-09-20T18:12:43.528 DEBUG:teuthology.orchestra.run.smithi174:> rm -fr /tmp/tmp.bM4Tl8OYB0 2024-09-20T18:12:43.543 DEBUG:teuthology.orchestra.run.smithi174:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T18:12:43.606 INFO:teuthology.orchestra.run.smithi174.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T18:12:43.606 INFO:teuthology.orchestra.run.smithi174.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T18:12:43.606 INFO:teuthology.orchestra.run.smithi174.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T18:12:43.606 INFO:teuthology.orchestra.run.smithi174.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T18:12:43.607 DEBUG:teuthology.orchestra.run.smithi174:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T18:12:43.758 DEBUG:teuthology.orchestra.run.smithi174:> sudo shutdown -r now 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout:Installed: 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:12:44.250 INFO:teuthology.orchestra.run.smithi012.stdout:Complete! 2024-09-20T18:12:44.618 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T18:12:44.722 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T18:12:44.722 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T18:12:44.722 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-20T18:12:44.722 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-20T18:12:44.722 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-20T18:12:44.722 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-20T18:12:44.722 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi012.front.sepia.ceph.com, path=None, version=distro) 2024-09-20T18:12:44.722 DEBUG:teuthology.orchestra.run.smithi012:> rpm -q kernel | sort -rV | head -n 1 2024-09-20T18:12:44.780 INFO:teuthology.orchestra.run.smithi012.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-20T18:12:44.781 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-20T18:12:44.781 DEBUG:teuthology.orchestra.run.smithi012:> sudo rpm -qi grub2-tools 2024-09-20T18:12:44.812 INFO:teuthology.orchestra.run.smithi012.stdout:Name : grub2-tools 2024-09-20T18:12:44.812 INFO:teuthology.orchestra.run.smithi012.stdout:Epoch : 1 2024-09-20T18:12:44.812 INFO:teuthology.orchestra.run.smithi012.stdout:Version : 2.06 2024-09-20T18:12:44.812 INFO:teuthology.orchestra.run.smithi012.stdout:Release : 82.el9 2024-09-20T18:12:44.812 INFO:teuthology.orchestra.run.smithi012.stdout:Architecture: x86_64 2024-09-20T18:12:44.812 INFO:teuthology.orchestra.run.smithi012.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-20T18:12:44.812 INFO:teuthology.orchestra.run.smithi012.stdout:Group : Unspecified 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Size : 8274670 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:License : GPLv3+ 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Packager : builder@centos.org 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Vendor : CentOS 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Summary : Support tools for GRUB. 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:Description : 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-20T18:12:44.813 INFO:teuthology.orchestra.run.smithi012.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-20T18:12:44.814 INFO:teuthology.orchestra.run.smithi012.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-20T18:12:44.814 INFO:teuthology.orchestra.run.smithi012.stdout:hardware devices. 2024-09-20T18:12:44.814 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:12:44.814 INFO:teuthology.orchestra.run.smithi012.stdout:This subpackage provides tools for support of all platforms. 2024-09-20T18:12:44.815 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-20T18:12:44.815 INFO:teuthology.task.kernel:Updating grub on smithi012 to boot 5.14.0-508.el9.x86_64 2024-09-20T18:12:44.815 DEBUG:teuthology.orchestra.run.smithi012:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-20T18:12:45.450 INFO:teuthology.orchestra.run.smithi012.stderr:Generating grub configuration file ... 2024-09-20T18:12:46.714 INFO:teuthology.orchestra.run.smithi012.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-20T18:12:46.747 INFO:teuthology.orchestra.run.smithi012.stderr:done 2024-09-20T18:12:46.749 DEBUG:teuthology.orchestra.run.smithi012:> mktemp 2024-09-20T18:12:46.803 INFO:teuthology.orchestra.run.smithi012.stdout:/tmp/tmp.dAKXrmd2Hw 2024-09-20T18:12:46.803 DEBUG:teuthology.orchestra.run.smithi012:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.dAKXrmd2Hw 2024-09-20T18:12:46.828 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0666 /tmp/tmp.dAKXrmd2Hw 2024-09-20T18:12:46.968 DEBUG:teuthology.orchestra.remote:smithi012:/tmp/tmp.dAKXrmd2Hw is 6KB 2024-09-20T18:12:47.016 DEBUG:teuthology.orchestra.run.smithi012:> rm -fr /tmp/tmp.dAKXrmd2Hw 2024-09-20T18:12:47.030 DEBUG:teuthology.orchestra.run.smithi012:> sudo /bin/ls /boot/loader/entries || true 2024-09-20T18:12:47.092 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-20T18:12:47.092 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-20T18:12:47.092 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-20T18:12:47.092 INFO:teuthology.orchestra.run.smithi012.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-20T18:12:47.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-20T18:12:47.240 DEBUG:teuthology.orchestra.run.smithi012:> sudo shutdown -r now 2024-09-20T18:13:13.786 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-20T18:13:13.787 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-09-20T18:13:13.787 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:13:17.246 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-20T18:13:17.247 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-20T18:13:17.247 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:14:13.843 DEBUG:teuthology.orchestra.remote:timed out 2024-09-20T18:14:17.250 DEBUG:teuthology.orchestra.remote:timed out 2024-09-20T18:14:22.848 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-09-20T18:14:22.849 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:14:24.145 DEBUG:teuthology.orchestra.run.smithi174:> true 2024-09-20T18:14:24.716 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-09-20T18:14:24.716 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-20T18:14:24.716 DEBUG:teuthology.orchestra.run.smithi174:> uname -r 2024-09-20T18:14:24.730 INFO:teuthology.orchestra.run.smithi174.stdout:5.14.0-508.el9.x86_64 2024-09-20T18:14:24.730 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T18:14:24.731 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T18:14:24.731 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T18:14:25.732 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-20T18:14:25.732 DEBUG:teuthology.orchestra.run.smithi174:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T18:14:25.825 INFO:teuthology.orchestra.run.smithi174.stdout:ttyS1 2024-09-20T18:14:25.853 DEBUG:teuthology.parallel:result is None 2024-09-20T18:14:26.251 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-20T18:14:26.252 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:14:26.729 DEBUG:teuthology.orchestra.run.smithi012:> true 2024-09-20T18:14:27.133 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-20T18:14:27.133 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-20T18:14:27.133 DEBUG:teuthology.orchestra.run.smithi012:> uname -r 2024-09-20T18:14:27.148 INFO:teuthology.orchestra.run.smithi012.stdout:5.14.0-508.el9.x86_64 2024-09-20T18:14:27.148 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-20T18:14:27.148 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-20T18:14:27.148 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-20T18:14:28.150 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-20T18:14:28.150 DEBUG:teuthology.orchestra.run.smithi012:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-20T18:14:28.230 INFO:teuthology.orchestra.run.smithi012.stdout:ttyS1 2024-09-20T18:14:28.254 DEBUG:teuthology.parallel:result is None 2024-09-20T18:14:28.255 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-20T18:14:28.261 INFO:teuthology.task.internal:Creating test directory... 2024-09-20T18:14:28.261 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T18:14:28.263 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-20T18:14:28.279 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-20T18:14:28.308 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-20T18:14:28.349 INFO:teuthology.task.internal:Creating archive directory... 2024-09-20T18:14:28.349 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T18:14:28.351 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-20T18:14:28.380 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-20T18:14:28.389 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-20T18:14:28.389 DEBUG:teuthology.orchestra.run.smithi012:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T18:14:28.419 DEBUG:teuthology.orchestra.run.smithi174:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-20T18:14:28.455 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T18:14:28.459 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T18:14:28.465 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T18:14:28.471 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-20T18:14:28.473 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-20T18:14:28.480 INFO:teuthology.task.internal:Configuring sudo... 2024-09-20T18:14:28.480 DEBUG:teuthology.orchestra.run.smithi012:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T18:14:28.526 DEBUG:teuthology.orchestra.run.smithi174:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-20T18:14:28.552 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-20T18:14:28.560 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-20T18:14:28.560 DEBUG:teuthology.orchestra.run.smithi012:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T18:14:28.594 DEBUG:teuthology.orchestra.run.smithi174:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-20T18:14:28.609 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T18:14:28.696 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T18:14:28.750 DEBUG:teuthology.orchestra.run.smithi012:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T18:14:28.835 DEBUG:teuthology.orchestra.run.smithi012:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T18:14:28.876 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:14:28.876 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T18:14:28.943 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T18:14:28.986 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-20T18:14:29.023 DEBUG:teuthology.orchestra.run.smithi174:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T18:14:29.100 DEBUG:teuthology.orchestra.run.smithi174:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-20T18:14:29.141 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-09-20T18:14:29.141 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-20T18:14:29.203 DEBUG:teuthology.orchestra.run.smithi012:> sudo service rsyslog restart 2024-09-20T18:14:29.206 DEBUG:teuthology.orchestra.run.smithi174:> sudo service rsyslog restart 2024-09-20T18:14:29.247 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T18:14:29.292 INFO:teuthology.orchestra.run.smithi174.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T18:14:29.765 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-20T18:14:29.773 INFO:teuthology.task.internal:Starting timer... 2024-09-20T18:14:29.773 INFO:teuthology.run_tasks:Running task pcp... 2024-09-20T18:14:29.782 INFO:teuthology.run_tasks:Running task selinux... 2024-09-20T18:14:29.789 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-20T18:14:29.789 DEBUG:teuthology.orchestra.run.smithi012:> sudo service auditd rotate 2024-09-20T18:14:29.867 INFO:teuthology.orchestra.run.smithi012.stdout:Rotating logs: 2024-09-20T18:14:29.869 DEBUG:teuthology.orchestra.run.smithi174:> sudo service auditd rotate 2024-09-20T18:14:29.924 INFO:teuthology.orchestra.run.smithi174.stdout:Rotating logs: 2024-09-20T18:14:29.926 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-20T18:14:29.926 DEBUG:teuthology.orchestra.run.smithi012:> /usr/sbin/getenforce 2024-09-20T18:14:29.968 INFO:teuthology.orchestra.run.smithi012.stdout:Permissive 2024-09-20T18:14:29.968 DEBUG:teuthology.orchestra.run.smithi174:> /usr/sbin/getenforce 2024-09-20T18:14:29.992 INFO:teuthology.orchestra.run.smithi174.stdout:Permissive 2024-09-20T18:14:29.992 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi012.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi174.front.sepia.ceph.com': 'permissive'} 2024-09-20T18:14:29.992 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T18:14:30.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:14:30.037 DEBUG:teuthology.orchestra.run.smithi174:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T18:14:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:14:30.061 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-20T18:14:30.061 DEBUG:teuthology.orchestra.run.smithi012:> sudo /usr/sbin/setenforce permissive 2024-09-20T18:14:30.115 DEBUG:teuthology.orchestra.run.smithi174:> sudo /usr/sbin/setenforce permissive 2024-09-20T18:14:30.224 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-20T18:14:30.233 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-20T18:14:30.244 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-20T18:14:30.245 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi012.front.sepia.ceph.com,smithi174.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-20T18:18:34.793 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi012.front.sepia.ceph.com'), Remote(name='ubuntu@smithi174.front.sepia.ceph.com')] 2024-09-20T18:18:34.794 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-20T18:18:34.795 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi012.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:18:34.882 DEBUG:teuthology.orchestra.run.smithi012:> true 2024-09-20T18:18:34.963 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi012.front.sepia.ceph.com' 2024-09-20T18:18:34.963 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-09-20T18:18:34.963 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi174.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-20T18:18:35.047 DEBUG:teuthology.orchestra.run.smithi174:> true 2024-09-20T18:18:35.121 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi174.front.sepia.ceph.com' 2024-09-20T18:18:35.121 INFO:teuthology.run_tasks:Running task clock... 2024-09-20T18:18:35.135 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-20T18:18:35.135 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T18:18:35.136 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T18:18:35.138 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-20T18:18:35.139 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T18:18:35.171 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T18:18:35.185 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T18:18:35.192 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-20T18:18:35.206 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-20T18:18:35.212 INFO:teuthology.orchestra.run.smithi012.stderr:sudo: ntpd: command not found 2024-09-20T18:18:35.224 INFO:teuthology.orchestra.run.smithi012.stdout:506 Cannot talk to daemon 2024-09-20T18:18:35.232 INFO:teuthology.orchestra.run.smithi174.stderr:sudo: ntpd: command not found 2024-09-20T18:18:35.238 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T18:18:35.243 INFO:teuthology.orchestra.run.smithi174.stdout:506 Cannot talk to daemon 2024-09-20T18:18:35.256 INFO:teuthology.orchestra.run.smithi012.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T18:18:35.258 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-20T18:18:35.273 INFO:teuthology.orchestra.run.smithi174.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-20T18:18:35.317 INFO:teuthology.orchestra.run.smithi012.stderr:bash: line 1: ntpq: command not found 2024-09-20T18:18:35.321 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T18:18:35.321 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2024-09-20T18:18:35.321 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.321 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.321 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.321 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.329 INFO:teuthology.orchestra.run.smithi174.stderr:bash: line 1: ntpq: command not found 2024-09-20T18:18:35.332 INFO:teuthology.orchestra.run.smithi174.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T18:18:35.332 INFO:teuthology.orchestra.run.smithi174.stdout:=============================================================================== 2024-09-20T18:18:35.332 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.332 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.332 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.332 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:18:35.333 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-20T18:18:35.452 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '88deadb8082f4a33832e228ccefc59cedba158a5'} 2024-09-20T18:18:35.452 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-20T18:18:35.452 INFO:tasks.cephadm:Cluster fsid is c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:18:35.452 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-20T18:18:35.452 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-20T18:18:35.453 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi012': '172.21.15.12', 'mon.smithi174': '172.21.15.174'} 2024-09-20T18:18:35.453 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-20T18:18:35.453 DEBUG:teuthology.orchestra.run.smithi012:> sudo hostname $(hostname -s) 2024-09-20T18:18:35.480 DEBUG:teuthology.orchestra.run.smithi174:> sudo hostname $(hostname -s) 2024-09-20T18:18:35.506 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-20T18:18:35.507 DEBUG:teuthology.orchestra.run.smithi012:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T18:18:35.781 INFO:teuthology.orchestra.run.smithi012.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 18:18 /home/ubuntu/cephtest/cephadm 2024-09-20T18:18:35.782 DEBUG:teuthology.orchestra.run.smithi174:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-20T18:18:36.104 INFO:teuthology.orchestra.run.smithi174.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 20 18:18 /home/ubuntu/cephtest/cephadm 2024-09-20T18:18:36.104 DEBUG:teuthology.orchestra.run.smithi012:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T18:18:36.123 DEBUG:teuthology.orchestra.run.smithi174:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-20T18:18:36.178 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-20T18:18:36.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T18:18:36.187 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-20T18:18:37.722 INFO:teuthology.orchestra.run.smithi012.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T18:18:37.767 INFO:teuthology.orchestra.run.smithi174.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T18:18:49.502 INFO:teuthology.orchestra.run.smithi174.stdout:{ 2024-09-20T18:18:49.503 INFO:teuthology.orchestra.run.smithi174.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T18:18:49.503 INFO:teuthology.orchestra.run.smithi174.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T18:18:49.503 INFO:teuthology.orchestra.run.smithi174.stdout: "repo_digests": [ 2024-09-20T18:18:49.503 INFO:teuthology.orchestra.run.smithi174.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T18:18:49.503 INFO:teuthology.orchestra.run.smithi174.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T18:18:49.503 INFO:teuthology.orchestra.run.smithi174.stdout: ] 2024-09-20T18:18:49.503 INFO:teuthology.orchestra.run.smithi174.stdout:} 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout: "repo_digests": [ 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-20T18:18:51.720 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:18:51.739 DEBUG:teuthology.orchestra.run.smithi012:> sudo mkdir -p /etc/ceph 2024-09-20T18:18:51.787 DEBUG:teuthology.orchestra.run.smithi174:> sudo mkdir -p /etc/ceph 2024-09-20T18:18:51.822 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 777 /etc/ceph 2024-09-20T18:18:51.854 DEBUG:teuthology.orchestra.run.smithi174:> sudo chmod 777 /etc/ceph 2024-09-20T18:18:51.893 INFO:tasks.cephadm:Writing seed config... 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-20T18:18:51.894 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-20T18:18:51.895 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:18:51.895 DEBUG:teuthology.orchestra.run.smithi012:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-20T18:18:51.913 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = c01c9b48-777c-11ef-baf5-efdc52797490 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-20T18:18:51.913 DEBUG:teuthology.orchestra.run.smithi012:mon.smithi012> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service 2024-09-20T18:18:51.957 INFO:tasks.cephadm:Bootstrapping... 2024-09-20T18:18:51.957 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid c01c9b48-777c-11ef-baf5-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.12 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-20T18:18:52.155 INFO:teuthology.orchestra.run.smithi012.stderr:-------------------------------------------------------------------------------- 2024-09-20T18:18:52.156 INFO:teuthology.orchestra.run.smithi012.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'c01c9b48-777c-11ef-baf5-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.12', '--skip-admin-label'] 2024-09-20T18:18:52.177 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: 5.2.2 2024-09-20T18:18:52.181 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying podman|docker is present... 2024-09-20T18:18:52.203 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: 5.2.2 2024-09-20T18:18:52.203 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying lvm2 is present... 2024-09-20T18:18:52.203 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying time synchronization is in place... 2024-09-20T18:18:52.215 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T18:18:52.217 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2024-09-20T18:18:52.224 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: enabled 2024-09-20T18:18:52.230 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: active 2024-09-20T18:18:52.231 INFO:teuthology.orchestra.run.smithi012.stderr:Unit chronyd.service is enabled and running 2024-09-20T18:18:52.231 INFO:teuthology.orchestra.run.smithi012.stderr:Repeating the final host check... 2024-09-20T18:18:52.252 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: 5.2.2 2024-09-20T18:18:52.254 INFO:teuthology.orchestra.run.smithi012.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-20T18:18:52.254 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl is present 2024-09-20T18:18:52.255 INFO:teuthology.orchestra.run.smithi012.stderr:lvcreate is present 2024-09-20T18:18:52.260 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-20T18:18:52.267 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2024-09-20T18:18:52.274 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: enabled 2024-09-20T18:18:52.280 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: active 2024-09-20T18:18:52.281 INFO:teuthology.orchestra.run.smithi012.stderr:Unit chronyd.service is enabled and running 2024-09-20T18:18:52.281 INFO:teuthology.orchestra.run.smithi012.stderr:Host looks OK 2024-09-20T18:18:52.281 INFO:teuthology.orchestra.run.smithi012.stderr:Cluster fsid: c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:18:52.282 INFO:teuthology.orchestra.run.smithi012.stderr:Acquiring lock 140337042114448 on /run/cephadm/c01c9b48-777c-11ef-baf5-efdc52797490.lock 2024-09-20T18:18:52.282 INFO:teuthology.orchestra.run.smithi012.stderr:Lock 140337042114448 acquired on /run/cephadm/c01c9b48-777c-11ef-baf5-efdc52797490.lock 2024-09-20T18:18:52.282 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying IP 172.21.15.12 port 3300 ... 2024-09-20T18:18:52.282 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying IP 172.21.15.12 port 6789 ... 2024-09-20T18:18:52.282 INFO:teuthology.orchestra.run.smithi012.stderr:Base mon IP is 172.21.15.12, final addrv is [v2:172.21.15.12:3300,v1:172.21.15.12:6789] 2024-09-20T18:18:52.285 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.12 metric 100 2024-09-20T18:18:52.285 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.12 metric 100 2024-09-20T18:18:52.290 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-20T18:18:52.290 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-20T18:18:52.290 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1785sec hoplimit 64 pref medium 2024-09-20T18:18:52.290 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-20T18:18:52.290 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-20T18:18:52.291 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T18:18:52.291 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-20T18:18:52.291 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:25ea/64 scope link 2024-09-20T18:18:52.291 INFO:teuthology.orchestra.run.smithi012.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-20T18:18:52.291 INFO:teuthology.orchestra.run.smithi012.stderr:Mon IP `172.21.15.12` is in CIDR network `172.21.0.0/20` 2024-09-20T18:18:52.291 INFO:teuthology.orchestra.run.smithi012.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-20T18:18:52.302 INFO:teuthology.orchestra.run.smithi012.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-20T18:18:52.395 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-20T18:18:52.774 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Getting image source signatures 2024-09-20T18:18:52.774 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-20T18:18:52.774 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-20T18:18:52.774 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-20T18:18:52.775 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-20T18:18:52.775 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-20T18:18:52.775 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T18:18:52.914 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: Writing manifest to image destination 2024-09-20T18:18:53.025 INFO:teuthology.orchestra.run.smithi012.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-20T18:18:53.431 INFO:teuthology.orchestra.run.smithi012.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T18:18:57.104 INFO:teuthology.orchestra.run.smithi012.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-20T18:18:57.105 INFO:teuthology.orchestra.run.smithi012.stderr:Extracting ceph user uid/gid from container image... 2024-09-20T18:19:00.803 INFO:teuthology.orchestra.run.smithi012.stderr:stat: 167 167 2024-09-20T18:19:03.926 INFO:teuthology.orchestra.run.smithi012.stderr:Creating initial keys... 2024-09-20T18:19:04.298 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-authtool: AQCYvO1m35S4DBAAgU4xiNgEGZmnUxjoFBs4Lg== 2024-09-20T18:19:05.453 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-authtool: AQCZvO1m31FYFhAA3qSCaKFjqcNKmZSrfO7ufA== 2024-09-20T18:19:06.325 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph-authtool: AQCavO1m/57JDRAAaWlgCfgq0Y1UIG5t3NuIwQ== 2024-09-20T18:19:06.793 INFO:teuthology.orchestra.run.smithi012.stderr:Creating initial monmap... 2024-09-20T18:19:07.173 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T18:19:07.173 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-20T18:19:07.173 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:07.173 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T18:19:07.721 INFO:teuthology.orchestra.run.smithi012.stderr:monmaptool for smithi012 [v2:172.21.15.12:3300,v1:172.21.15.12:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-20T18:19:07.722 INFO:teuthology.orchestra.run.smithi012.stderr:setting min_mon_release = octopus 2024-09-20T18:19:07.722 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: set fsid to c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:07.722 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-20T18:19:07.722 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:19:07.722 INFO:teuthology.orchestra.run.smithi012.stderr:Creating mon... 2024-09-20T18:19:08.618 INFO:teuthology.orchestra.run.smithi012.stderr:create mon.smithi012 on 2024-09-20T18:19:08.845 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-20T18:19:08.998 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-c01c9b48-777c-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-c01c9b48-777c-11ef-baf5-efdc52797490.target. 2024-09-20T18:19:08.998 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-c01c9b48-777c-11ef-baf5-efdc52797490.target → /etc/systemd/system/ceph-c01c9b48-777c-11ef-baf5-efdc52797490.target. 2024-09-20T18:19:09.530 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to reset failed state of unit ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service: Unit ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service not loaded. 2024-09-20T18:19:09.539 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c01c9b48-777c-11ef-baf5-efdc52797490.target.wants/ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service → /etc/systemd/system/ceph-c01c9b48-777c-11ef-baf5-efdc52797490@.service. 2024-09-20T18:19:10.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:10 smithi012 podman[23378]: 2024-09-20 18:19:10.023482958 +0000 UTC m=+0.016829711 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-20T18:19:11.234 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2024-09-20T18:19:11.241 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2024-09-20T18:19:11.241 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2024-09-20T18:19:11.242 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T18:19:11.242 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mon to start... 2024-09-20T18:19:11.242 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mon... 2024-09-20T18:19:11.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:10 smithi012 podman[23378]: 2024-09-20 18:19:10.964171655 +0000 UTC m=+0.957518440 container create 32062ad29e96b94ec5e6a70cbd06adbc6670ca076d771bbd43233a3c753d58dd (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, GIT_CLEAN=True, distribution-scope=public, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=) 2024-09-20T18:19:11.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:11 smithi012 podman[23378]: 2024-09-20 18:19:11.141384801 +0000 UTC m=+1.134731545 container init 32062ad29e96b94ec5e6a70cbd06adbc6670ca076d771bbd43233a3c753d58dd (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, vcs-type=git, distribution-scope=public, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8, ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-20T18:19:11.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:11 smithi012 podman[23378]: 2024-09-20 18:19:11.145001874 +0000 UTC m=+1.138348618 container start 32062ad29e96b94ec5e6a70cbd06adbc6670ca076d771bbd43233a3c753d58dd (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., name=centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, maintainer=Guillaume Abrioux , distribution-scope=public, io.openshift.tags=base centos centos-stream) 2024-09-20T18:19:11.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:11 smithi012 ceph-mon[23396]: mkfs c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:11.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:11 smithi012 ceph-mon[23396]: mon.smithi012 is new leader, mons smithi012 in quorum (ranks 0) 2024-09-20T18:19:11.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:11 smithi012 bash[23378]: 32062ad29e96b94ec5e6a70cbd06adbc6670ca076d771bbd43233a3c753d58dd 2024-09-20T18:19:11.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:11 smithi012 systemd[1]: Started Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: cluster: 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: id: c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: services: 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi012 (age 0.553116s) 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-20T18:19:11.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-20T18:19:11.752 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:11.752 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: data: 2024-09-20T18:19:11.752 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-20T18:19:11.752 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-20T18:19:11.752 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-20T18:19:11.752 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: pgs: 2024-09-20T18:19:11.752 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:12.344 INFO:teuthology.orchestra.run.smithi012.stderr:mon is available 2024-09-20T18:19:12.345 INFO:teuthology.orchestra.run.smithi012.stderr:Assimilating anything we can from ceph.conf... 2024-09-20T18:19:12.455 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:12 smithi012 ceph-mon[23396]: mon.smithi012 is new leader, mons smithi012 in quorum (ranks 0) 2024-09-20T18:19:12.455 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:12 smithi012 ceph-mon[23396]: monmap e1: 1 mons at {smithi012=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} 2024-09-20T18:19:12.455 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:12 smithi012 ceph-mon[23396]: fsmap 2024-09-20T18:19:12.455 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:12 smithi012 ceph-mon[23396]: osdmap e1: 0 total, 0 up, 0 in 2024-09-20T18:19:12.455 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:12 smithi012 ceph-mon[23396]: mgrmap e1: no daemons active 2024-09-20T18:19:12.455 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:12 smithi012 ceph-mon[23396]: from='client.? 172.21.15.12:0/1486927399' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-20T18:19:12.865 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:12.865 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [global] 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: fsid = c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.12:3300,v1:172.21.15.12:6789] 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [mgr] 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [osd] 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T18:19:12.866 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T18:19:13.446 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:13 smithi012 ceph-mon[23396]: from='client.? 172.21.15.12:0/557539668' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T18:19:13.446 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:13 smithi012 ceph-mon[23396]: from='client.? 172.21.15.12:0/557539668' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-20T18:19:13.446 INFO:teuthology.orchestra.run.smithi012.stderr:Generating new minimal ceph.conf... 2024-09-20T18:19:14.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:14 smithi012 ceph-mon[23396]: from='client.? 172.21.15.12:0/3271098269' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:19:14.491 INFO:teuthology.orchestra.run.smithi012.stderr:Restarting the monitor... 2024-09-20T18:19:14.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:14 smithi012 systemd[1]: Stopping Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:19:14.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:14 smithi012 bash[23614]: Error: no container with name or ID "ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon.smithi012" found: no such container 2024-09-20T18:19:14.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:14 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[23392]: 2024-09-20T18:19:14.646+0000 7fed7c7f0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi012 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:19:14.782 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:14 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[23392]: 2024-09-20T18:19:14.646+0000 7fed7c7f0700 -1 mon.smithi012@0(leader) e1 *** Got Signal Terminated *** 2024-09-20T18:19:15.048 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:14 smithi012 podman[23625]: 2024-09-20 18:19:14.770228116 +0000 UTC m=+0.225013539 container died 32062ad29e96b94ec5e6a70cbd06adbc6670ca076d771bbd43233a3c753d58dd (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_CLEAN=True, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-type=git, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870) 2024-09-20T18:19:15.342 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 podman[23625]: 2024-09-20 18:19:15.062583992 +0000 UTC m=+0.517369396 container cleanup 32062ad29e96b94ec5e6a70cbd06adbc6670ca076d771bbd43233a3c753d58dd (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, vcs-type=git, vendor=Red Hat, Inc., GIT_CLEAN=True, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, version=8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public) 2024-09-20T18:19:15.342 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 bash[23625]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012 2024-09-20T18:19:15.342 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 bash[23652]: Error: no container with name or ID "ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon.smithi012" found: no such container 2024-09-20T18:19:15.674 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 podman[23638]: 2024-09-20 18:19:15.330396782 +0000 UTC m=+0.553546265 container remove 32062ad29e96b94ec5e6a70cbd06adbc6670ca076d771bbd43233a3c753d58dd (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, GIT_BRANCH=HEAD, version=8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, ceph=True, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-20T18:19:15.674 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service: Deactivated successfully. 2024-09-20T18:19:15.674 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 systemd[1]: Stopped Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:19:15.674 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 systemd[1]: Starting Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:19:15.674 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 podman[23725]: 2024-09-20 18:19:15.469329794 +0000 UTC m=+0.017241067 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-20T18:19:15.674 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 podman[23725]: 2024-09-20 18:19:15.572796695 +0000 UTC m=+0.120707972 container create 15a06395140cc6effb9f83e38adb041262111cc9d89125ecd0aa3b4912940487 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, name=centos-stream, ceph=True, io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.19.8, version=8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64) 2024-09-20T18:19:15.815 INFO:teuthology.orchestra.run.smithi012.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-20T18:19:16.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 podman[23725]: 2024-09-20 18:19:15.742305142 +0000 UTC m=+0.290216403 container init 15a06395140cc6effb9f83e38adb041262111cc9d89125ecd0aa3b4912940487 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0) 2024-09-20T18:19:16.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 podman[23725]: 2024-09-20 18:19:15.745814487 +0000 UTC m=+0.293725748 container start 15a06395140cc6effb9f83e38adb041262111cc9d89125ecd0aa3b4912940487 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, distribution-scope=public, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.buildah.version=1.19.8, GIT_CLEAN=True) 2024-09-20T18:19:16.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T18:19:16.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-20T18:19:16.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: pidfile_write: ignore empty --pid-file 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: load: jerasure load: lrc 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: RocksDB version: 6.15.5 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Compile date Apr 18 2022 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: DB SUMMARY 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: DB Session ID: D2FDDQ5UGI2KWDIOGCWP 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: CURRENT file: CURRENT 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi012/store.db dir, Total Num: 1, files: 000008.sst 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi012/store.db: 000010.log size: 82707 ; 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.error_if_exists: 0 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.create_if_missing: 0 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.paranoid_checks: 1 2024-09-20T18:19:16.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.env: 0x5598caec6860 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.fs: Posix File System 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.info_log: 0x5598cd547fa0 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.statistics: (nil) 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.use_fsync: 0 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_log_file_size: 0 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T18:19:16.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T18:19:16.073 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.allow_fallocate: 1 2024-09-20T18:19:16.073 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T18:19:16.073 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T18:19:16.073 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.use_direct_reads: 0 2024-09-20T18:19:16.073 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.db_log_dir: 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi012/store.db 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.write_buffer_manager: 0x5598cd638330 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T18:19:16.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-20T18:19:16.075 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T18:19:16.075 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T18:19:16.075 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T18:19:16.075 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.unordered_write: 0 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.row_cache: None 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.wal_filter: None 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.preserve_deletes: 0 2024-09-20T18:19:16.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.two_write_queues: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.atomic_flush: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.log_readahead_size: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_background_jobs: 2 2024-09-20T18:19:16.077 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_background_compactions: -1 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_subcompactions: 1 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_open_files: -1 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T18:19:16.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_background_flushes: -1 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Compression algorithms supported: 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kZSTD supported: 0 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kXpressCompression supported: 0 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kLZ4Compression supported: 1 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kBZip2Compression supported: 0 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kZlibCompression supported: 1 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: kSnappyCompression supported: 1 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi012/store.db/MANIFEST-000009 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.merge_operator: 2024-09-20T18:19:16.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_filter: None 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5598cd513d00) 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: index_type: 0 2024-09-20T18:19:16.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: data_block_index_type: 0 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: index_shortening: 1 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: hash_index_allow_collision: 1 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: checksum: 1 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: no_block_cache: 0 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache: 0x5598cd57e170 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache_options: 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: capacity : 536870912 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: num_shard_bits : 4 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: strict_capacity_limit : 0 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache_compressed: (nil) 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: persistent_cache: (nil) 2024-09-20T18:19:16.081 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_size: 4096 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_size_deviation: 10 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_restart_interval: 16 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: index_block_restart_interval: 1 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: metadata_block_size: 4096 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: partition_filters: 0 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: use_delta_encoding: 1 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: whole_key_filtering: 1 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: verify_compression: 0 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: format_version: 4 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: enable_index_compression: 1 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_align: 0 2024-09-20T18:19:16.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T18:19:16.083 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T18:19:16.083 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression: NoCompression 2024-09-20T18:19:16.083 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T18:19:16.083 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T18:19:16.083 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.num_levels: 7 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T18:19:16.084 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T18:19:16.085 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.arena_block_size: 4194304 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T18:19:16.086 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.table_properties_collectors: 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.inplace_update_support: 0 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T18:19:16.087 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T18:19:16.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T18:19:16.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.bloom_locality: 0 2024-09-20T18:19:16.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.max_successive_merges: 0 2024-09-20T18:19:16.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T18:19:16.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T18:19:16.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T18:19:16.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T18:19:16.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.ttl: 2592000 2024-09-20T18:19:16.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T18:19:16.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.enable_blob_files: false 2024-09-20T18:19:16.090 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.min_blob_size: 0 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi012/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726856355773451, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-20T18:19:16.091 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726856355774214, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79389, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77653, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71962, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726856355, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "6d3e43f4-b963-4a4d-8e9d-c094fdd17e42", "db_session_id": "D2FDDQ5UGI2KWDIOGCWP"}} 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726856355774546, "job": 1, "event": "recovery_finished"} 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi012/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5598cd564a80 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: DB pointer 0x5598cd574000 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-20T18:19:16.094 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** DB Stats ** 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.9 0.00 0.00 1 0.000 0 0 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.9 0.00 0.00 1 0.000 0 0 2024-09-20T18:19:16.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.9 0.00 0.00 1 0.000 0 0 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 153.9 0.00 0.00 1 0.000 0 0 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T18:19:16.096 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T18:19:16.125 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative compaction: 0.00 GB write, 24.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:19:16.125 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval compaction: 0.00 GB write, 24.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.9 0.00 0.00 1 0.000 0 0 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.9 0.00 0.00 1 0.000 0 0 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-20T18:19:16.126 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 153.9 0.00 0.00 1 0.000 0 0 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative compaction: 0.00 GB write, 23.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: starting mon.smithi012 rank 0 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi012 fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:16.127 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???) e1 preinit fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???).mds e1 new map 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???).mds e1 print_map 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: e1 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: legacy client fscid: -1 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: No filesystems configured 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expand map: {default=false} 2024-09-20T18:19:16.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta from 'false' to 'false' 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expanded map: {default=false} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expand map: {default=info} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta from 'info' to 'info' 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expanded map: {default=info} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expand map: {default=daemon} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expanded map: {default=daemon} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expand map: {default=debug} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta from 'debug' to 'debug' 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: expand_channel_meta expanded map: {default=debug} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 bash[23725]: 15a06395140cc6effb9f83e38adb041262111cc9d89125ecd0aa3b4912940487 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 systemd[1]: Started Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mon.smithi012 is new leader, mons smithi012 in quorum (ranks 0) 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: monmap e1: 1 mons at {smithi012=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0]} 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: fsmap 2024-09-20T18:19:16.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: osdmap e1: 0 total, 0 up, 0 in 2024-09-20T18:19:16.130 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:15 smithi012 ceph-mon[23745]: mgrmap e1: no daemons active 2024-09-20T18:19:16.896 INFO:teuthology.orchestra.run.smithi012.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-20T18:19:16.896 INFO:teuthology.orchestra.run.smithi012.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-20T18:19:16.896 INFO:teuthology.orchestra.run.smithi012.stderr:Creating mgr... 2024-09-20T18:19:16.897 INFO:teuthology.orchestra.run.smithi012.stderr:Verifying port 9283 ... 2024-09-20T18:19:17.177 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Failed to reset failed state of unit ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mgr.smithi012.xkxxrd.service: Unit ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mgr.smithi012.xkxxrd.service not loaded. 2024-09-20T18:19:17.185 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: Created symlink /etc/systemd/system/ceph-c01c9b48-777c-11ef-baf5-efdc52797490.target.wants/ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mgr.smithi012.xkxxrd.service → /etc/systemd/system/ceph-c01c9b48-777c-11ef-baf5-efdc52797490@.service. 2024-09-20T18:19:17.471 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:17 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1534100092' entity='client.admin' 2024-09-20T18:19:18.152 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2024-09-20T18:19:18.159 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2024-09-20T18:19:18.160 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2024-09-20T18:19:18.160 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to enable service . firewalld.service is not available 2024-09-20T18:19:18.168 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2024-09-20T18:19:18.176 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2024-09-20T18:19:18.177 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2024-09-20T18:19:18.177 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-20T18:19:18.177 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr to start... 2024-09-20T18:19:18.177 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr... 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "c01c9b48-777c-11ef-baf5-efdc52797490", 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:18.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "smithi012" 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:18.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T18:19:18.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:18.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T18:19:18.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T18:19:18.739 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:18.740 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T18:19:18.741 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": false, 2024-09-20T18:19:18.741 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T18:19:18.741 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T18:19:18.741 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2024-09-20T18:19:18.741 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2024-09-20T18:19:18.750 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2024-09-20T18:19:18.750 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:18.750 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:18.750 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:18.750 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T18:19:18.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:18.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2024-09-20T18:19:11.187096+0000", 2024-09-20T18:19:18.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:18.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:18.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T18:19:18.751 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:19:19.207 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:18 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1567751752' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T18:19:19.207 INFO:teuthology.orchestra.run.smithi012.stderr:mgr not available, waiting (1/15)... 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "c01c9b48-777c-11ef-baf5-efdc52797490", 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:21.731 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "smithi012" 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T18:19:21.732 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T18:19:21.736 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": false, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2024-09-20T18:19:21.737 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2024-09-20T18:19:11.187096+0000", 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T18:19:21.738 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:19:21.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:21 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2858734138' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T18:19:22.269 INFO:teuthology.orchestra.run.smithi012.stderr:mgr not available, waiting (2/15)... 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: Activating manager daemon smithi012.xkxxrd 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: mgrmap e2: smithi012.xkxxrd(active, starting, since 0.00289527s) 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:19:24.678 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:24.679 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:24.679 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:24 smithi012 ceph-mon[23745]: from='mgr.14100 172.21.15.12:0/63282037' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:24.777 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:24.777 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:19:24.777 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "c01c9b48-777c-11ef-baf5-efdc52797490", 2024-09-20T18:19:24.777 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2024-09-20T18:19:24.777 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T18:19:24.777 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "smithi012" 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T18:19:24.778 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T18:19:24.779 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": false, 2024-09-20T18:19:24.780 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T18:19:24.781 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:24.782 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2024-09-20T18:19:11.187096+0000", 2024-09-20T18:19:24.782 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:24.782 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:24.782 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T18:19:24.782 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:19:25.309 INFO:teuthology.orchestra.run.smithi012.stderr:mgr not available, waiting (3/15)... 2024-09-20T18:19:25.681 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:25 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/781452191' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T18:19:25.681 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:25 smithi012 ceph-mon[23745]: mgrmap e3: smithi012.xkxxrd(active, since 1.00711s) 2024-09-20T18:19:27.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:27 smithi012 ceph-mon[23745]: mgrmap e4: smithi012.xkxxrd(active, since 2s) 2024-09-20T18:19:28.017 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:28.017 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:19:28.017 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsid": "c01c9b48-777c-11ef-baf5-efdc52797490", 2024-09-20T18:19:28.017 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "health": { 2024-09-20T18:19:28.017 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "checks": {}, 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mutes": [] 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum": [ 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 0 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "smithi012" 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "monmap": { 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:28.018 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osdmap": { 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgmap": { 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-20T18:19:28.019 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "fsmap": { 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": true, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modules": [ 2024-09-20T18:19:28.020 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "iostat", 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "nfs", 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "restful" 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ], 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "servicemap": { 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "modified": "2024-09-20T18:19:11.187096+0000", 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "services": {} 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: }, 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-20T18:19:28.021 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:19:28.614 INFO:teuthology.orchestra.run.smithi012.stderr:mgr is available 2024-09-20T18:19:28.998 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:28 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3940762092' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-20T18:19:29.217 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:29.217 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [global] 2024-09-20T18:19:29.217 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: fsid = c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:29.217 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-20T18:19:29.217 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [mgr] 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: [osd] 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-20T18:19:29.218 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-20T18:19:29.770 INFO:teuthology.orchestra.run.smithi012.stderr:Enabling cephadm module... 2024-09-20T18:19:30.022 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:29 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/704475008' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-20T18:19:31.114 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:30 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2669019407' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-20T18:19:31.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:31 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2669019407' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-20T18:19:31.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:31 smithi012 ceph-mon[23745]: mgrmap e5: smithi012.xkxxrd(active, since 6s) 2024-09-20T18:19:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:19:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-20T18:19:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": true, 2024-09-20T18:19:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "active_name": "smithi012.xkxxrd", 2024-09-20T18:19:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T18:19:32.080 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:19:32.651 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for the mgr to restart... 2024-09-20T18:19:32.651 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr epoch 5... 2024-09-20T18:19:32.846 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:32 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1010625490' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T18:19:38.229 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:19:38.229 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-20T18:19:38.229 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T18:19:38.229 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:19:38.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: Active manager daemon smithi012.xkxxrd restarted 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: Activating manager daemon smithi012.xkxxrd 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: osdmap e2: 0 total, 0 up, 0 in 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: mgrmap e6: smithi012.xkxxrd(active, starting, since 0.0524235s) 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:19:38.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:19:38.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:19:38.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:19:38.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:38 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:19:38.846 INFO:teuthology.orchestra.run.smithi012.stderr:mgr epoch 5 is available 2024-09-20T18:19:38.846 INFO:teuthology.orchestra.run.smithi012.stderr:Setting orchestrator backend to cephadm... 2024-09-20T18:19:39.453 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:39 smithi012 ceph-mon[23745]: mgrmap e7: smithi012.xkxxrd(active, since 1.05518s) 2024-09-20T18:19:39.453 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:39 smithi012 ceph-mon[23745]: [20/Sep/2024:18:19:38] ENGINE Bus STARTING 2024-09-20T18:19:39.453 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:39 smithi012 ceph-mon[23745]: [20/Sep/2024:18:19:38] ENGINE Serving on https://172.21.15.12:7150 2024-09-20T18:19:39.454 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:39 smithi012 ceph-mon[23745]: [20/Sep/2024:18:19:38] ENGINE Bus STARTED 2024-09-20T18:19:39.454 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:39 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:39.454 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:39 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:19:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:40 smithi012 ceph-mon[23745]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:40 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:40 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:19:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:40 smithi012 ceph-mon[23745]: mgrmap e8: smithi012.xkxxrd(active, since 2s) 2024-09-20T18:19:40.616 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: value unchanged 2024-09-20T18:19:41.141 INFO:teuthology.orchestra.run.smithi012.stderr:Generating ssh key... 2024-09-20T18:19:42.073 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:41 smithi012 ceph-mon[23745]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:42.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:42 smithi012 ceph-mon[23745]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:42.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:42 smithi012 ceph-mon[23745]: Generating ssh key... 2024-09-20T18:19:42.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:42 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:42.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:42 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:43.196 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC24E9Vegha2EsZO6ryNr4LeFTAGYSpznzzXKifMvWuj1h5Phq/9exCnNpNtA5PtImRo94E+Cqug33pjgyHZLfopqsoFt41JN5l9D2kHWbY1usRHG6jTEos4aeUzpnBHQ7r4v2RKIWoLiM3P4mcEk0LCi/CJCfJIq3ahjrYCsnQz/FSuqXWI0qvHw2X1qm13bEAI4oKA6WoHtCm3XEpUPwbKUdGoImx7VcOj850jLr8Q7AnzCN0AwqJ1lh80Kk3icaC2Ve5K3ON6Pke3tddGxbUDmbjeTi+7VrBZf396vD53xznl+iPNudXlUTKc1YiZ4wFKBwluBgJYsCpZ260O1fEdU/bwtmoNxuzDKDHibfEKTIuwpdCAyRBBCW5Lh9yS1TVUj5892AjMZCYGwwTA5SztjoAgJN2c8oN/9Udog7ruAW/MQgi/yEMZ5WF1a2aCyRcVHQ1zwlskwOgXOzX/A1V9RktgxebxnYewp7pFqgYZDJJ948eJaY+hGAE/CxpxXc= ceph-c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:19:43.796 INFO:teuthology.orchestra.run.smithi012.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-20T18:19:43.796 INFO:teuthology.orchestra.run.smithi012.stderr:Adding key to root@localhost authorized_keys... 2024-09-20T18:19:43.802 INFO:teuthology.orchestra.run.smithi012.stderr:Adding host smithi012... 2024-09-20T18:19:44.061 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:43 smithi012 ceph-mon[23745]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:45.108 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:44 smithi012 ceph-mon[23745]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi012", "addr": "172.21.15.12", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:45.108 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:44 smithi012 ceph-mon[23745]: Deploying cephadm binary to smithi012 2024-09-20T18:19:45.350 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Added host 'smithi012' with addr '172.21.15.12' 2024-09-20T18:19:45.968 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying mon service with default placement... 2024-09-20T18:19:46.603 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:46 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:46.603 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:46 smithi012 ceph-mon[23745]: Added host smithi012 2024-09-20T18:19:46.603 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:46 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:19:46.603 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-20T18:19:47.322 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying mgr service with default placement... 2024-09-20T18:19:47.853 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:47 smithi012 ceph-mon[23745]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:47.853 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:47 smithi012 ceph-mon[23745]: Saving service mon spec with placement count:5 2024-09-20T18:19:47.853 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:47 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:48.087 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-20T18:19:48.862 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying crash service with default placement... 2024-09-20T18:19:49.176 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:49 smithi012 ceph-mon[23745]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:49.177 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:49 smithi012 ceph-mon[23745]: Saving service mgr spec with placement count:2 2024-09-20T18:19:49.177 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:49 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:49.889 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-20T18:19:50.715 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying prometheus service with default placement... 2024-09-20T18:19:50.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:50 smithi012 ceph-mon[23745]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:50.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:50 smithi012 ceph-mon[23745]: Saving service crash spec with placement * 2024-09-20T18:19:50.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:50 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:50.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:50 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:51.389 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-20T18:19:51.971 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying grafana service with default placement... 2024-09-20T18:19:52.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:52 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:52.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:52 smithi012 ceph-mon[23745]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:52.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:52 smithi012 ceph-mon[23745]: Saving service prometheus spec with placement count:1 2024-09-20T18:19:52.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:52 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:52.610 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-20T18:19:53.705 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:53 smithi012 ceph-mon[23745]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:53.705 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:53 smithi012 ceph-mon[23745]: Saving service grafana spec with placement count:1 2024-09-20T18:19:53.705 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:53 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:53.705 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:53 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:53.707 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying node-exporter service with default placement... 2024-09-20T18:19:54.394 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-20T18:19:54.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:54 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:54.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:54 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:55.876 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:55 smithi012 ceph-mon[23745]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:55.876 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:55 smithi012 ceph-mon[23745]: Saving service node-exporter spec with placement * 2024-09-20T18:19:55.882 INFO:teuthology.orchestra.run.smithi012.stderr:Deploying alertmanager service with default placement... 2024-09-20T18:19:56.507 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-20T18:19:57.721 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:57 smithi012 ceph-mon[23745]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:19:57.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:57 smithi012 ceph-mon[23745]: Saving service alertmanager spec with placement count:1 2024-09-20T18:19:57.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:57 smithi012 ceph-mon[23745]: from='mgr.14120 172.21.15.12:0/376668229' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:19:58.868 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:58 smithi012 ceph-mon[23745]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:19:58.868 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:19:58 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2962658262' entity='client.admin' 2024-09-20T18:19:59.724 INFO:teuthology.orchestra.run.smithi012.stderr:Enabling the dashboard module... 2024-09-20T18:20:00.338 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:00 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2348771580' entity='client.admin' 2024-09-20T18:20:00.338 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:00 smithi012 ceph-mon[23745]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:01.364 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:01 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/868513503' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-20T18:20:02.199 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:02 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/868513503' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-20T18:20:02.199 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:02 smithi012 ceph-mon[23745]: mgrmap e9: smithi012.xkxxrd(active, since 23s) 2024-09-20T18:20:02.440 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:20:02.440 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-20T18:20:02.440 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "available": true, 2024-09-20T18:20:02.440 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "active_name": "smithi012.xkxxrd", 2024-09-20T18:20:02.440 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-20T18:20:02.440 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:20:03.198 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for the mgr to restart... 2024-09-20T18:20:03.198 INFO:teuthology.orchestra.run.smithi012.stderr:Waiting for mgr epoch 9... 2024-09-20T18:20:03.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:03 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/4294526070' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: Active manager daemon smithi012.xkxxrd restarted 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: Activating manager daemon smithi012.xkxxrd 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: osdmap e3: 0 total, 0 up, 0 in 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: mgrmap e10: smithi012.xkxxrd(active, starting, since 0.0528094s) 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:20:08.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:20:08.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:20:08.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:20:08.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:20:08.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:20:08.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:08 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:20:08.524 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: { 2024-09-20T18:20:08.525 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-20T18:20:08.525 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: "initialized": true 2024-09-20T18:20:08.525 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: } 2024-09-20T18:20:09.274 INFO:teuthology.orchestra.run.smithi012.stderr:mgr epoch 9 is available 2024-09-20T18:20:09.274 INFO:teuthology.orchestra.run.smithi012.stderr:Generating a dashboard self-signed certificate... 2024-09-20T18:20:09.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:09 smithi012 ceph-mon[23745]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-20T18:20:09.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:09 smithi012 ceph-mon[23745]: mgrmap e11: smithi012.xkxxrd(active, since 1.05563s) 2024-09-20T18:20:09.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:09 smithi012 ceph-mon[23745]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-20T18:20:09.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:09 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:09.971 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-20T18:20:10.623 INFO:teuthology.orchestra.run.smithi012.stderr:Creating initial admin user... 2024-09-20T18:20:10.949 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: [20/Sep/2024:18:20:09] ENGINE Bus STARTING 2024-09-20T18:20:10.949 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: [20/Sep/2024:18:20:09] ENGINE Serving on https://172.21.15.12:7150 2024-09-20T18:20:10.949 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: [20/Sep/2024:18:20:09] ENGINE Bus STARTED 2024-09-20T18:20:10.950 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:10.950 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:20:10.950 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:10.950 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:10.950 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: mgrmap e12: smithi012.xkxxrd(active, since 2s) 2024-09-20T18:20:10.950 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:10 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:11.509 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$Gs8svEw7dOqnFCf54Y0J/OvhFtObtLEuN2oRK62m/lkSFsbxDehCe", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726856411, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-20T18:20:12.352 INFO:teuthology.orchestra.run.smithi012.stderr:Fetching dashboard port number... 2024-09-20T18:20:12.614 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:12 smithi012 ceph-mon[23745]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:20:12.614 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:12 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:13.419 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: 8443 2024-09-20T18:20:13.721 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:13 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1400836758' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-20T18:20:14.094 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: disabled 2024-09-20T18:20:14.097 INFO:teuthology.orchestra.run.smithi012.stderr:systemctl: inactive 2024-09-20T18:20:14.098 INFO:teuthology.orchestra.run.smithi012.stderr:firewalld.service is not enabled 2024-09-20T18:20:14.098 INFO:teuthology.orchestra.run.smithi012.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-20T18:20:14.101 INFO:teuthology.orchestra.run.smithi012.stderr:Ceph Dashboard is now available at: 2024-09-20T18:20:14.101 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:14.101 INFO:teuthology.orchestra.run.smithi012.stderr: URL: https://smithi012.front.sepia.ceph.com:8443/ 2024-09-20T18:20:14.101 INFO:teuthology.orchestra.run.smithi012.stderr: User: admin 2024-09-20T18:20:14.101 INFO:teuthology.orchestra.run.smithi012.stderr: Password: g9jrpa8aof 2024-09-20T18:20:14.101 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:14.101 INFO:teuthology.orchestra.run.smithi012.stderr:Enabling autotune for osd_memory_target 2024-09-20T18:20:14.721 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:14 smithi012 ceph-mon[23745]: mgrmap e13: smithi012.xkxxrd(active, since 6s) 2024-09-20T18:20:16.196 INFO:teuthology.orchestra.run.smithi012.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-20T18:20:16.818 INFO:teuthology.orchestra.run.smithi012.stderr:You can access the Ceph CLI with: 2024-09-20T18:20:16.818 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:16.818 INFO:teuthology.orchestra.run.smithi012.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-20T18:20:16.818 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:16.818 INFO:teuthology.orchestra.run.smithi012.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-20T18:20:16.818 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:16.819 INFO:teuthology.orchestra.run.smithi012.stderr: ceph telemetry on 2024-09-20T18:20:16.819 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:16.819 INFO:teuthology.orchestra.run.smithi012.stderr:For more information see: 2024-09-20T18:20:16.819 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:16.819 INFO:teuthology.orchestra.run.smithi012.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-20T18:20:16.819 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:20:16.819 INFO:teuthology.orchestra.run.smithi012.stderr:Bootstrap complete. 2024-09-20T18:20:16.868 INFO:tasks.cephadm:Fetching config... 2024-09-20T18:20:16.868 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:20:16.868 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-20T18:20:16.887 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-20T18:20:16.887 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:20:16.887 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-20T18:20:16.943 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-20T18:20:16.944 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:20:16.944 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/keyring of=/dev/stdout 2024-09-20T18:20:17.011 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-20T18:20:17.012 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:20:17.012 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-20T18:20:17.068 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-20T18:20:17.068 DEBUG:teuthology.orchestra.run.smithi012:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC24E9Vegha2EsZO6ryNr4LeFTAGYSpznzzXKifMvWuj1h5Phq/9exCnNpNtA5PtImRo94E+Cqug33pjgyHZLfopqsoFt41JN5l9D2kHWbY1usRHG6jTEos4aeUzpnBHQ7r4v2RKIWoLiM3P4mcEk0LCi/CJCfJIq3ahjrYCsnQz/FSuqXWI0qvHw2X1qm13bEAI4oKA6WoHtCm3XEpUPwbKUdGoImx7VcOj850jLr8Q7AnzCN0AwqJ1lh80Kk3icaC2Ve5K3ON6Pke3tddGxbUDmbjeTi+7VrBZf396vD53xznl+iPNudXlUTKc1YiZ4wFKBwluBgJYsCpZ260O1fEdU/bwtmoNxuzDKDHibfEKTIuwpdCAyRBBCW5Lh9yS1TVUj5892AjMZCYGwwTA5SztjoAgJN2c8oN/9Udog7ruAW/MQgi/yEMZ5WF1a2aCyRcVHQ1zwlskwOgXOzX/A1V9RktgxebxnYewp7pFqgYZDJJ948eJaY+hGAE/CxpxXc= ceph-c01c9b48-777c-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T18:20:17.153 INFO:teuthology.orchestra.run.smithi012.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC24E9Vegha2EsZO6ryNr4LeFTAGYSpznzzXKifMvWuj1h5Phq/9exCnNpNtA5PtImRo94E+Cqug33pjgyHZLfopqsoFt41JN5l9D2kHWbY1usRHG6jTEos4aeUzpnBHQ7r4v2RKIWoLiM3P4mcEk0LCi/CJCfJIq3ahjrYCsnQz/FSuqXWI0qvHw2X1qm13bEAI4oKA6WoHtCm3XEpUPwbKUdGoImx7VcOj850jLr8Q7AnzCN0AwqJ1lh80Kk3icaC2Ve5K3ON6Pke3tddGxbUDmbjeTi+7VrBZf396vD53xznl+iPNudXlUTKc1YiZ4wFKBwluBgJYsCpZ260O1fEdU/bwtmoNxuzDKDHibfEKTIuwpdCAyRBBCW5Lh9yS1TVUj5892AjMZCYGwwTA5SztjoAgJN2c8oN/9Udog7ruAW/MQgi/yEMZ5WF1a2aCyRcVHQ1zwlskwOgXOzX/A1V9RktgxebxnYewp7pFqgYZDJJ948eJaY+hGAE/CxpxXc= ceph-c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:20:17.162 DEBUG:teuthology.orchestra.run.smithi174:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC24E9Vegha2EsZO6ryNr4LeFTAGYSpznzzXKifMvWuj1h5Phq/9exCnNpNtA5PtImRo94E+Cqug33pjgyHZLfopqsoFt41JN5l9D2kHWbY1usRHG6jTEos4aeUzpnBHQ7r4v2RKIWoLiM3P4mcEk0LCi/CJCfJIq3ahjrYCsnQz/FSuqXWI0qvHw2X1qm13bEAI4oKA6WoHtCm3XEpUPwbKUdGoImx7VcOj850jLr8Q7AnzCN0AwqJ1lh80Kk3icaC2Ve5K3ON6Pke3tddGxbUDmbjeTi+7VrBZf396vD53xznl+iPNudXlUTKc1YiZ4wFKBwluBgJYsCpZ260O1fEdU/bwtmoNxuzDKDHibfEKTIuwpdCAyRBBCW5Lh9yS1TVUj5892AjMZCYGwwTA5SztjoAgJN2c8oN/9Udog7ruAW/MQgi/yEMZ5WF1a2aCyRcVHQ1zwlskwOgXOzX/A1V9RktgxebxnYewp7pFqgYZDJJ948eJaY+hGAE/CxpxXc= ceph-c01c9b48-777c-11ef-baf5-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-20T18:20:17.238 INFO:teuthology.orchestra.run.smithi174.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC24E9Vegha2EsZO6ryNr4LeFTAGYSpznzzXKifMvWuj1h5Phq/9exCnNpNtA5PtImRo94E+Cqug33pjgyHZLfopqsoFt41JN5l9D2kHWbY1usRHG6jTEos4aeUzpnBHQ7r4v2RKIWoLiM3P4mcEk0LCi/CJCfJIq3ahjrYCsnQz/FSuqXWI0qvHw2X1qm13bEAI4oKA6WoHtCm3XEpUPwbKUdGoImx7VcOj850jLr8Q7AnzCN0AwqJ1lh80Kk3icaC2Ve5K3ON6Pke3tddGxbUDmbjeTi+7VrBZf396vD53xznl+iPNudXlUTKc1YiZ4wFKBwluBgJYsCpZ260O1fEdU/bwtmoNxuzDKDHibfEKTIuwpdCAyRBBCW5Lh9yS1TVUj5892AjMZCYGwwTA5SztjoAgJN2c8oN/9Udog7ruAW/MQgi/yEMZ5WF1a2aCyRcVHQ1zwlskwOgXOzX/A1V9RktgxebxnYewp7pFqgYZDJJ948eJaY+hGAE/CxpxXc= ceph-c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:20:17.250 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-20T18:20:17.471 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:17 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/874870761' entity='client.admin' 2024-09-20T18:20:21.662 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-20T18:20:21.662 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-20T18:20:21.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:21 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2190792928' entity='client.admin' 2024-09-20T18:20:21.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:21 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:21.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:21 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi012", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:20:21.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:21 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:21.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:21 smithi012 ceph-mon[23745]: Deploying daemon alertmanager.smithi012 on smithi012 2024-09-20T18:20:24.606 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi174 2024-09-20T18:20:24.606 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-09-20T18:20:24.606 DEBUG:teuthology.orchestra.run.smithi174:> dd of=/etc/ceph/ceph.conf 2024-09-20T18:20:24.625 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-09-20T18:20:24.625 DEBUG:teuthology.orchestra.run.smithi174:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:20:24.685 INFO:tasks.cephadm:Adding host smithi174 to orchestrator... 2024-09-20T18:20:24.685 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch host add smithi174 2024-09-20T18:20:24.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:24 smithi012 ceph-mon[23745]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:20:24.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:24 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:28.685 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:28 smithi012 ceph-mon[23745]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:30.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:30 smithi012 ceph-mon[23745]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:31.438 INFO:teuthology.orchestra.run.smithi012.stdout:Added host 'smithi174' with addr '172.21.15.174' 2024-09-20T18:20:32.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:31 smithi012 ceph-mon[23745]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi174", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:20:32.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:31 smithi012 ceph-mon[23745]: Deploying cephadm binary to smithi174 2024-09-20T18:20:32.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:31 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:32.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:31 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T18:20:32.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:31 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-20T18:20:32.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:31 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:20:32.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:31 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:32.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch host ls --format=json 2024-09-20T18:20:32.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:32 smithi012 ceph-mon[23745]: Deploying daemon crash.smithi012 on smithi012 2024-09-20T18:20:32.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:32 smithi012 ceph-mon[23745]: Added host smithi174 2024-09-20T18:20:32.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:32 smithi012 ceph-mon[23745]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:32.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:32 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:34.906 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:34 smithi012 ceph-mon[23745]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:35.360 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:20:35.360 INFO:teuthology.orchestra.run.smithi012.stdout:[{"addr": "172.21.15.12", "hostname": "smithi012", "labels": [], "status": ""}, {"addr": "172.21.15.174", "hostname": "smithi174", "labels": [], "status": ""}] 2024-09-20T18:20:36.292 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-20T18:20:36.292 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd crush tunables default 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T18:20:37.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:37.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:36 smithi012 ceph-mon[23745]: Deploying daemon grafana.smithi012 on smithi012 2024-09-20T18:20:38.895 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:38 smithi012 ceph-mon[23745]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:38.895 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:38 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:38.895 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:38 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2474219876' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-20T18:20:38.895 INFO:teuthology.orchestra.run.smithi012.stderr:adjusted tunables profile to default 2024-09-20T18:20:39.589 INFO:tasks.cephadm:Adding mon.smithi012 on smithi012 2024-09-20T18:20:39.589 INFO:tasks.cephadm:Adding mon.smithi174 on smithi174 2024-09-20T18:20:39.589 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch apply mon '2;smithi012:172.21.15.12=smithi012;smithi174:172.21.15.174=smithi174' 2024-09-20T18:20:40.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:39 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2474219876' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-20T18:20:40.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:39 smithi012 ceph-mon[23745]: osdmap e4: 0 total, 0 up, 0 in 2024-09-20T18:20:41.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:40 smithi012 ceph-mon[23745]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:41.430 INFO:teuthology.orchestra.run.smithi174.stdout:Scheduled mon update... 2024-09-20T18:20:41.946 DEBUG:teuthology.orchestra.run.smithi174:mon.smithi174> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi174.service 2024-09-20T18:20:41.948 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:20:41.948 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:20:42.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:42 smithi012 ceph-mon[23745]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi012:172.21.15.12=smithi012;smithi174:172.21.15.174=smithi174", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:20:42.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:42 smithi012 ceph-mon[23745]: Saving service mon spec with placement smithi012:172.21.15.12=smithi012;smithi174:172.21.15.174=smithi174;count:2 2024-09-20T18:20:42.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:42 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:42.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:42 smithi012 ceph-mon[23745]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:43.830 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:20:43.830 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:20:43.832 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:20:44.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:44 smithi012 ceph-mon[23745]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:44.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:44 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/4053471417' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:20:45.403 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:20:45.404 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:20:47.057 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:20:47.057 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:20:47.060 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:20:47.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:46 smithi012 ceph-mon[23745]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:48.045 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:47 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/3889200369' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:20:48.639 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:20:48.639 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:20:49.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:49 smithi012 ceph-mon[23745]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:49.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:49 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:49.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:49 smithi012 ceph-mon[23745]: Deploying daemon node-exporter.smithi012 on smithi012 2024-09-20T18:20:50.453 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:20:50.454 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:20:50.456 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:20:51.288 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:51 smithi012 ceph-mon[23745]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:51.288 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:51 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/233371496' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:20:52.014 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:20:52.014 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:20:52.765 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:52 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:20:52.765 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:52 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-20T18:20:52.766 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:52 smithi012 ceph-mon[23745]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:20:53.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:53 smithi012 ceph-mon[23745]: from='mgr.14162 172.21.15.12:0/2738444668' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-20T18:20:53.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:53 smithi012 ceph-mon[23745]: mgrmap e14: smithi012.xkxxrd(active, since 45s) 2024-09-20T18:20:53.813 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:20:53.813 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:20:53.815 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:20:54.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:54 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/1730435051' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:20:55.418 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:20:55.419 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:20:57.191 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:20:57.191 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:20:57.193 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:20:57.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:57 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/3637684952' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:20:58.902 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:20:58.902 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:21:00.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: Active manager daemon smithi012.xkxxrd restarted 2024-09-20T18:21:00.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: Activating manager daemon smithi012.xkxxrd 2024-09-20T18:21:00.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: osdmap e5: 0 total, 0 up, 0 in 2024-09-20T18:21:00.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: mgrmap e15: smithi012.xkxxrd(active, starting, since 0.0533798s) 2024-09-20T18:21:00.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: [20/Sep/2024:18:20:59] ENGINE Bus STARTING 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: [20/Sep/2024:18:20:59] ENGINE Serving on https://172.21.15.12:7150 2024-09-20T18:21:00.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: [20/Sep/2024:18:20:59] ENGINE Bus STARTED 2024-09-20T18:21:00.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:20:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:00.680 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:21:00.680 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:21:00.682 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:21:01.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:00 smithi012 ceph-mon[23745]: mgrmap e16: smithi012.xkxxrd(active, since 1.05629s) 2024-09-20T18:21:01.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:01.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:00 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/2970285354' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:21:01.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:02.289 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:21:02.290 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:21:02.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:02 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:02.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:02 smithi012 ceph-mon[23745]: mgrmap e17: smithi012.xkxxrd(active, since 2s) 2024-09-20T18:21:03.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:03.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:03.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi012", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:21:03.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi174", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-20T18:21:04.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:05.414 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:21:05.414 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:21:05.417 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:21:05.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:05 smithi012 ceph-mon[23745]: Deploying daemon crash.smithi174 on smithi174 2024-09-20T18:21:05.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:05 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/4204684183' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:21:07.754 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:21:07.755 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:21:08.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:08 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:08.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:08 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:21:08.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:08 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-20T18:21:08.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:08 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:21:08.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:08 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:08.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:08 smithi012 ceph-mon[23745]: Deploying daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:21:10.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:10.699 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:21:10.700 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:21:10.701 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:21:11.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:10 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/687310685' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:21:12.399 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:21:12.399 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:21:12.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:12.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T18:21:12.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:12.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:12 smithi012 ceph-mon[23745]: Deploying daemon mon.smithi174 on smithi174 2024-09-20T18:21:15.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:15.714 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:21:15.714 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":1,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:19:07.084846Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-20T18:21:15.715 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 1 2024-09-20T18:21:16.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:15 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/3532145671' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:21:17.525 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-20T18:21:17.525 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mon dump -f json 2024-09-20T18:21:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:21:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:21:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: mon.smithi012 calling monitor election 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/715734993' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: mon.smithi174 calling monitor election 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: mon.smithi012 is new leader, mons smithi012,smithi174 in quorum (ranks 0,1) 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: monmap e2: 2 mons at {smithi012=[v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0],smithi174=[v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0]} 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: fsmap 2024-09-20T18:21:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: osdmap e5: 0 total, 0 up, 0 in 2024-09-20T18:21:21.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: mgrmap e17: smithi012.xkxxrd(active, since 22s) 2024-09-20T18:21:21.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:21:21.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/715734993' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:21:21.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: overall HEALTH_OK 2024-09-20T18:21:21.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/715734993' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:21:21.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/715734993' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:21:21.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:22.430 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:21:22.430 INFO:teuthology.orchestra.run.smithi174.stdout:{"epoch":2,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","modified":"2024-09-20T18:21:16.095481Z","created":"2024-09-20T18:19:07.084846Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi012","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:3300","nonce":0},{"type":"v1","addr":"172.21.15.12:6789","nonce":0}]},"addr":"172.21.15.12:6789/0","public_addr":"172.21.15.12:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi174","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:3300","nonce":0},{"type":"v1","addr":"172.21.15.174:6789","nonce":0}]},"addr":"172.21.15.174:6789/0","public_addr":"172.21.15.174:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-20T18:21:22.431 INFO:teuthology.orchestra.run.smithi174.stderr:dumped monmap epoch 2 2024-09-20T18:21:22.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:22 smithi012 ceph-mon[23745]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:22.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:22 smithi012 ceph-mon[23745]: Deploying daemon node-exporter.smithi174 on smithi174 2024-09-20T18:21:22.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:22 smithi012 ceph-mon[23745]: mgrmap e18: smithi012.xkxxrd(active, since 22s), standbys: smithi174.nxkptu 2024-09-20T18:21:22.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:22 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi174.nxkptu", "id": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:21:22.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:22 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:21:23.241 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-20T18:21:23.241 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph config generate-minimal-conf 2024-09-20T18:21:23.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:23 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/3485241231' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-20T18:21:24.463 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:24 smithi012 ceph-mon[23745]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:24.464 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:24 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:25.118 INFO:teuthology.orchestra.run.smithi012.stdout:# minimal ceph.conf for c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:21:25.119 INFO:teuthology.orchestra.run.smithi012.stdout:[global] 2024-09-20T18:21:25.119 INFO:teuthology.orchestra.run.smithi012.stdout: fsid = c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:21:25.119 INFO:teuthology.orchestra.run.smithi012.stdout: mon_host = [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] 2024-09-20T18:21:25.852 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-20T18:21:25.852 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:21:25.852 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T18:21:25.881 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:21:25.881 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:21:25.952 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-09-20T18:21:25.952 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.conf 2024-09-20T18:21:25.985 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-09-20T18:21:25.986 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:21:26.054 DEBUG:tasks.cephadm:set 0 configs 2024-09-20T18:21:26.054 INFO:tasks.cephadm:Deploying OSDs... 2024-09-20T18:21:26.055 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:21:26.055 DEBUG:teuthology.orchestra.run.smithi012:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T18:21:26.071 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T18:21:26.071 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_1 2024-09-20T18:21:26.127 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T18:21:26.127 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:26.127 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-20T18:21:26.128 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:26.128 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:26.128 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-20 18:20:48.685692025 +0000 2024-09-20T18:21:26.128 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-20 18:18:12.422196900 +0000 2024-09-20T18:21:26.128 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-20 18:18:12.422196900 +0000 2024-09-20T18:21:26.128 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-20 18:18:12.422196900 +0000 2024-09-20T18:21:26.128 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T18:21:26.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:26 smithi012 ceph-mon[23745]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:26.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:26 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:26.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:26 smithi012 ceph-mon[23745]: Deploying daemon prometheus.smithi012 on smithi012 2024-09-20T18:21:26.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:26 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2311473617' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:26.192 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-20T18:21:26.192 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-20T18:21:26.192 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000214075 s, 2.4 MB/s 2024-09-20T18:21:26.193 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T18:21:26.250 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_2 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-20 18:20:48.685692025 +0000 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-20 18:18:12.761193487 +0000 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-20 18:18:12.761193487 +0000 2024-09-20T18:21:26.307 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-20 18:18:12.761193487 +0000 2024-09-20T18:21:26.308 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T18:21:26.371 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-20T18:21:26.372 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-20T18:21:26.372 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000220111 s, 2.3 MB/s 2024-09-20T18:21:26.373 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T18:21:26.430 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_3 2024-09-20T18:21:26.486 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T18:21:26.486 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:26.486 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-20T18:21:26.487 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:26.487 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:26.487 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-20 18:20:48.686692015 +0000 2024-09-20T18:21:26.487 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-20 18:18:13.101190064 +0000 2024-09-20T18:21:26.487 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-20 18:18:13.101190064 +0000 2024-09-20T18:21:26.487 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-20 18:18:13.101190064 +0000 2024-09-20T18:21:26.488 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T18:21:26.550 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-20T18:21:26.550 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-20T18:21:26.550 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000220981 s, 2.3 MB/s 2024-09-20T18:21:26.552 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T18:21:26.611 DEBUG:teuthology.orchestra.run.smithi012:> stat /dev/vg_nvme/lv_4 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout:Access: 2024-09-20 18:20:48.686692015 +0000 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout:Modify: 2024-09-20 18:18:13.469186359 +0000 2024-09-20T18:21:26.667 INFO:teuthology.orchestra.run.smithi012.stdout:Change: 2024-09-20 18:18:13.469186359 +0000 2024-09-20T18:21:26.668 INFO:teuthology.orchestra.run.smithi012.stdout: Birth: 2024-09-20 18:18:13.469186359 +0000 2024-09-20T18:21:26.668 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T18:21:26.731 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records in 2024-09-20T18:21:26.731 INFO:teuthology.orchestra.run.smithi012.stderr:1+0 records out 2024-09-20T18:21:26.731 INFO:teuthology.orchestra.run.smithi012.stderr:512 bytes copied, 0.000185507 s, 2.8 MB/s 2024-09-20T18:21:26.732 DEBUG:teuthology.orchestra.run.smithi012:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T18:21:26.789 DEBUG:teuthology.orchestra.run.smithi174:> set -ex 2024-09-20T18:21:26.789 DEBUG:teuthology.orchestra.run.smithi174:> dd if=/scratch_devs of=/dev/stdout 2024-09-20T18:21:26.809 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-20T18:21:26.809 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_1 2024-09-20T18:21:26.867 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-20T18:21:26.867 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:26.868 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-20T18:21:26.868 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:26.868 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:26.868 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-09-20 18:21:21.689650012 +0000 2024-09-20T18:21:26.868 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-09-20 18:18:13.635890192 +0000 2024-09-20T18:21:26.868 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-09-20 18:18:13.635890192 +0000 2024-09-20T18:21:26.868 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-09-20 18:18:13.635890192 +0000 2024-09-20T18:21:26.868 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-20T18:21:26.935 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-09-20T18:21:26.935 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-09-20T18:21:26.935 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000295503 s, 1.7 MB/s 2024-09-20T18:21:26.936 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-20T18:21:27.008 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_2 2024-09-20T18:21:27.065 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-09-20 18:21:21.690649999 +0000 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-09-20 18:18:13.963885886 +0000 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-09-20 18:18:13.963885886 +0000 2024-09-20T18:21:27.066 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-09-20 18:18:13.963885886 +0000 2024-09-20T18:21:27.066 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-20T18:21:27.133 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-09-20T18:21:27.133 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-09-20T18:21:27.133 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000258035 s, 2.0 MB/s 2024-09-20T18:21:27.134 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-20T18:21:27.193 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_3 2024-09-20T18:21:27.250 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-20T18:21:27.250 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:27.250 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-20T18:21:27.250 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:27.250 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:27.250 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-09-20 18:21:21.690649999 +0000 2024-09-20T18:21:27.251 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-09-20 18:18:14.313881291 +0000 2024-09-20T18:21:27.251 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-09-20 18:18:14.313881291 +0000 2024-09-20T18:21:27.251 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-09-20 18:18:14.313881291 +0000 2024-09-20T18:21:27.251 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-20T18:21:27.317 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-09-20T18:21:27.317 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-09-20T18:21:27.317 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000241492 s, 2.1 MB/s 2024-09-20T18:21:27.318 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-20T18:21:27.375 DEBUG:teuthology.orchestra.run.smithi174:> stat /dev/vg_nvme/lv_4 2024-09-20T18:21:27.431 INFO:teuthology.orchestra.run.smithi174.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-20T18:21:27.431 INFO:teuthology.orchestra.run.smithi174.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-20T18:21:27.431 INFO:teuthology.orchestra.run.smithi174.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-20T18:21:27.431 INFO:teuthology.orchestra.run.smithi174.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-20T18:21:27.431 INFO:teuthology.orchestra.run.smithi174.stdout:Context: system_u:object_r:device_t:s0 2024-09-20T18:21:27.431 INFO:teuthology.orchestra.run.smithi174.stdout:Access: 2024-09-20 18:21:21.690649999 +0000 2024-09-20T18:21:27.432 INFO:teuthology.orchestra.run.smithi174.stdout:Modify: 2024-09-20 18:18:14.661876723 +0000 2024-09-20T18:21:27.432 INFO:teuthology.orchestra.run.smithi174.stdout:Change: 2024-09-20 18:18:14.661876723 +0000 2024-09-20T18:21:27.432 INFO:teuthology.orchestra.run.smithi174.stdout: Birth: 2024-09-20 18:18:14.661876723 +0000 2024-09-20T18:21:27.432 DEBUG:teuthology.orchestra.run.smithi174:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-20T18:21:27.494 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records in 2024-09-20T18:21:27.494 INFO:teuthology.orchestra.run.smithi174.stderr:1+0 records out 2024-09-20T18:21:27.494 INFO:teuthology.orchestra.run.smithi174.stderr:512 bytes copied, 0.000231582 s, 2.2 MB/s 2024-09-20T18:21:27.495 DEBUG:teuthology.orchestra.run.smithi174:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-20T18:21:27.551 INFO:tasks.cephadm:Deploying osd.0 on smithi012 with /dev/vg_nvme/lv_4... 2024-09-20T18:21:27.551 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T18:21:28.362 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:28 smithi012 ceph-mon[23745]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:28.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:28 smithi174 ceph-mon[25682]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:30.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:29 smithi012 ceph-mon[23745]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:30.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:30.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:29 smithi174 ceph-mon[25682]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:30.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:32.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:31 smithi012 ceph-mon[23745]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:32.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:31 smithi174 ceph-mon[25682]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:34.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:33 smithi012 ceph-mon[23745]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:34.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:33 smithi174 ceph-mon[25682]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:36.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:35 smithi012 ceph-mon[23745]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:36.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:35 smithi174 ceph-mon[25682]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:37.904 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:37.905 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:21:37.905 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:37.905 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:21:37.905 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:37 smithi012 ceph-mon[23745]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:38.102 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:38.102 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:21:38.102 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:38.102 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:21:38.103 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:37 smithi174 ceph-mon[25682]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:38.624 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:21:38.645 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi012:vg_nvme/lv_4 2024-09-20T18:21:39.690 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:39 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:39.690 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:39 smithi012 ceph-mon[23745]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:21:39.690 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:39 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:39.691 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:39 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:39.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:39 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:39.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:39 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:21:39.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:39 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:39.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:39 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:40.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:40 smithi012 ceph-mon[23745]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:21:40.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:40 smithi012 ceph-mon[23745]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:40.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:40 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:21:40.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:40 smithi174 ceph-mon[25682]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:42.106 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:41 smithi012 ceph-mon[23745]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:42.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:41 smithi174 ceph-mon[25682]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:44.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:43 smithi012 ceph-mon[23745]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:44.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:43 smithi012 ceph-mon[23745]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:21:44.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:43 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:21:44.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:43 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:21:44.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:43 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:44.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:43 smithi174 ceph-mon[25682]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:44.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:43 smithi174 ceph-mon[25682]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:21:44.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:43 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:21:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:43 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:21:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:43 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:45.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:45 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:45.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:45 smithi012 ceph-mon[23745]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:21:45.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:45 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:45.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:45 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:45.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:45 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T18:21:45.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:45 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T18:21:45.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:45 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:45.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:45 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:45.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:45 smithi174 ceph-mon[25682]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:21:45.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:45 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:45.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:45 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:45.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:45 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T18:21:45.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:45 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T18:21:45.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:45 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:46.439 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:46 smithi012 ceph-mon[23745]: Reconfiguring mon.smithi012 (unknown last config time)... 2024-09-20T18:21:46.439 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:46 smithi012 ceph-mon[23745]: Reconfiguring daemon mon.smithi012 on smithi012 2024-09-20T18:21:46.439 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:46 smithi012 ceph-mon[23745]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:46.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:46 smithi174 ceph-mon[25682]: Reconfiguring mon.smithi012 (unknown last config time)... 2024-09-20T18:21:46.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:46 smithi174 ceph-mon[25682]: Reconfiguring daemon mon.smithi012 on smithi012 2024-09-20T18:21:46.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:46 smithi174 ceph-mon[25682]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:47.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:47.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:47 smithi012 ceph-mon[23745]: Reconfiguring mgr.smithi012.xkxxrd (unknown last config time)... 2024-09-20T18:21:47.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:21:47.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:21:47.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:47.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:47 smithi012 ceph-mon[23745]: Reconfiguring daemon mgr.smithi012.xkxxrd on smithi012 2024-09-20T18:21:47.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:47.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:47 smithi174 ceph-mon[25682]: Reconfiguring mgr.smithi012.xkxxrd (unknown last config time)... 2024-09-20T18:21:47.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:21:47.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:21:47.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:47.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:47 smithi174 ceph-mon[25682]: Reconfiguring daemon mgr.smithi012.xkxxrd on smithi012 2024-09-20T18:21:48.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:48 smithi012 ceph-mon[23745]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:48.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:48 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1714131983' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "123d122c-8976-4ecd-890a-396e77e8a687"}]: dispatch 2024-09-20T18:21:48.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:48 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1714131983' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "123d122c-8976-4ecd-890a-396e77e8a687"}]': finished 2024-09-20T18:21:48.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:48 smithi012 ceph-mon[23745]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T18:21:48.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:48 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:21:48.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:48 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1733653146' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:21:48.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:48 smithi174 ceph-mon[25682]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:48 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1714131983' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "123d122c-8976-4ecd-890a-396e77e8a687"}]: dispatch 2024-09-20T18:21:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:48 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1714131983' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "123d122c-8976-4ecd-890a-396e77e8a687"}]': finished 2024-09-20T18:21:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:48 smithi174 ceph-mon[25682]: osdmap e6: 1 total, 0 up, 1 in 2024-09-20T18:21:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:48 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:21:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:48 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1733653146' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:21:50.665 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:50 smithi012 ceph-mon[23745]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:50.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:50 smithi174 ceph-mon[25682]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:52.113 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:51 smithi012 ceph-mon[23745]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:52.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:51 smithi174 ceph-mon[25682]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:53.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:53.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:53 smithi012 ceph-mon[23745]: Reconfiguring alertmanager.smithi012 (dependencies changed)... 2024-09-20T18:21:53.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:53 smithi012 ceph-mon[23745]: Reconfiguring daemon alertmanager.smithi012 on smithi012 2024-09-20T18:21:53.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:53.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:53 smithi174 ceph-mon[25682]: Reconfiguring alertmanager.smithi012 (dependencies changed)... 2024-09-20T18:21:53.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:53 smithi174 ceph-mon[25682]: Reconfiguring daemon alertmanager.smithi012 on smithi012 2024-09-20T18:21:54.385 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:54 smithi012 ceph-mon[23745]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:54.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:54 smithi174 ceph-mon[25682]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:56.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:56 smithi012 ceph-mon[23745]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:56.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:56 smithi174 ceph-mon[25682]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:58.293 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:58 smithi012 ceph-mon[23745]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:58.294 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:58 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:58.294 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:58 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T18:21:58.294 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:58 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:58.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:58 smithi174 ceph-mon[25682]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:21:58.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:58 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:21:58.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:58 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T18:21:58.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:58 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:21:59.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:59 smithi012 ceph-mon[23745]: Reconfiguring crash.smithi012 (monmap changed)... 2024-09-20T18:21:59.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:59 smithi012 ceph-mon[23745]: Reconfiguring daemon crash.smithi012 on smithi012 2024-09-20T18:21:59.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:21:59.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:21:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:21:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:59 smithi174 ceph-mon[25682]: Reconfiguring crash.smithi012 (monmap changed)... 2024-09-20T18:21:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:59 smithi174 ceph-mon[25682]: Reconfiguring daemon crash.smithi012 on smithi012 2024-09-20T18:21:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:21:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:21:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:22:00.453 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:00 smithi012 ceph-mon[23745]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:00.453 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T18:22:00.453 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:00.453 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:00 smithi012 ceph-mon[23745]: Deploying daemon osd.0 on smithi012 2024-09-20T18:22:00.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:00 smithi174 ceph-mon[25682]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:00.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-20T18:22:00.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:00.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:00 smithi174 ceph-mon[25682]: Deploying daemon osd.0 on smithi012 2024-09-20T18:22:01.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:01 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:01.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:01 smithi012 ceph-mon[23745]: Reconfiguring grafana.smithi012 (dependencies changed)... 2024-09-20T18:22:01.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:01 smithi012 ceph-mon[23745]: Reconfiguring daemon grafana.smithi012 on smithi012 2024-09-20T18:22:01.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:01 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:01.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:01 smithi174 ceph-mon[25682]: Reconfiguring grafana.smithi012 (dependencies changed)... 2024-09-20T18:22:01.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:01 smithi174 ceph-mon[25682]: Reconfiguring daemon grafana.smithi012 on smithi012 2024-09-20T18:22:02.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:02 smithi012 ceph-mon[23745]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:02.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:02 smithi174 ceph-mon[25682]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:04 smithi012 ceph-mon[23745]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:04.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:04 smithi174 ceph-mon[25682]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:06.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:06 smithi012 ceph-mon[23745]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:06.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:06 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:06 smithi174 ceph-mon[25682]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:06 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:08.635 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:08 smithi012 ceph-mon[23745]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:08.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:08 smithi174 ceph-mon[25682]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:09.635 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:09 smithi012 ceph-mon[23745]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T18:22:09.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:09 smithi174 ceph-mon[25682]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-20T18:22:10.537 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:10 smithi012 ceph-mon[23745]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:10.537 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:10 smithi012 ceph-mon[23745]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T18:22:10.538 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:10 smithi012 ceph-mon[23745]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T18:22:10.538 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:10 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:10.538 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:10 smithi012 ceph-mon[23745]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:22:10.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:10 smithi174 ceph-mon[25682]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:10.739 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:10 smithi174 ceph-mon[25682]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T18:22:10.739 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:10 smithi174 ceph-mon[25682]: osdmap e7: 1 total, 0 up, 1 in 2024-09-20T18:22:10.739 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:10 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:10.739 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:10 smithi174 ceph-mon[25682]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:22:11.643 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:11 smithi012 ceph-mon[23745]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:22:11.644 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:11 smithi012 ceph-mon[23745]: osdmap e8: 1 total, 0 up, 1 in 2024-09-20T18:22:11.644 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:11.644 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:11.644 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:11.644 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T18:22:11.644 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:11.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:11 smithi174 ceph-mon[25682]: from='osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:22:11.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:11 smithi174 ceph-mon[25682]: osdmap e8: 1 total, 0 up, 1 in 2024-09-20T18:22:11.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:11.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:11.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:11.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-20T18:22:11.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:12.279 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 0 on host 'smithi012' 2024-09-20T18:22:12.501 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:22:12.501 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:22:12.501 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: Reconfiguring crash.smithi174 (monmap changed)... 2024-09-20T18:22:12.501 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: Reconfiguring daemon crash.smithi174 on smithi174 2024-09-20T18:22:12.501 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037] boot 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:12.502 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: Reconfiguring crash.smithi174 (monmap changed)... 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: Reconfiguring daemon crash.smithi174 on smithi174 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: osd.0 [v2:172.21.15.12:6802/459253037,v1:172.21.15.12:6803/459253037] boot 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: osdmap e9: 1 total, 1 up, 1 in 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:22:12.598 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:22:12.599 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:12.599 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:12.891 DEBUG:teuthology.orchestra.run.smithi012:osd.0> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.0.service 2024-09-20T18:22:12.894 INFO:tasks.cephadm:Deploying osd.1 on smithi012 with /dev/vg_nvme/lv_3... 2024-09-20T18:22:12.894 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T18:22:13.568 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:13 smithi174 ceph-mon[25682]: Reconfiguring mgr.smithi174.nxkptu (monmap changed)... 2024-09-20T18:22:13.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:13 smithi174 ceph-mon[25682]: Reconfiguring daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:22:13.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:13 smithi174 ceph-mon[25682]: osdmap e10: 1 total, 1 up, 1 in 2024-09-20T18:22:13.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:13.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T18:22:13.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T18:22:13.570 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:13.930 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:13 smithi012 ceph-mon[23745]: Reconfiguring mgr.smithi174.nxkptu (monmap changed)... 2024-09-20T18:22:13.930 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:13 smithi012 ceph-mon[23745]: Reconfiguring daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:22:13.930 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:13 smithi012 ceph-mon[23745]: osdmap e10: 1 total, 1 up, 1 in 2024-09-20T18:22:13.930 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:13.930 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T18:22:13.930 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T18:22:13.930 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:14.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:14.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: Reconfiguring mon.smithi174 (monmap changed)... 2024-09-20T18:22:14.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: Reconfiguring daemon mon.smithi174 on smithi174 2024-09-20T18:22:14.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:22:14.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.12:9093"}]: dispatch 2024-09-20T18:22:14.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:22:14.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.12:3000"}]: dispatch 2024-09-20T18:22:14.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:22:14.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.12:9095"}]: dispatch 2024-09-20T18:22:14.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:14.728 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:14.728 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:14.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: Reconfiguring mon.smithi174 (monmap changed)... 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: Reconfiguring daemon mon.smithi174 on smithi174 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.12:9093"}]: dispatch 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.12:3000"}]: dispatch 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.12:9095"}]: dispatch 2024-09-20T18:22:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:14.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:14.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:14.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:15.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:15 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:22:15.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:15 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.12:9093"}]: dispatch 2024-09-20T18:22:15.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:15 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:22:15.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:15 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.12:3000"}]: dispatch 2024-09-20T18:22:15.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:15 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:22:15.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:15 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.12:9095"}]: dispatch 2024-09-20T18:22:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:15 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:22:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:15 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.12:9093"}]: dispatch 2024-09-20T18:22:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:15 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:22:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:15 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.12:3000"}]: dispatch 2024-09-20T18:22:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:15 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:22:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:15 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.12:9095"}]: dispatch 2024-09-20T18:22:16.230 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:22:16.252 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi012:vg_nvme/lv_3 2024-09-20T18:22:16.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:16 smithi174 ceph-mon[25682]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:16 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:16.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:16 smithi012 ceph-mon[23745]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:16.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:16 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:18.631 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:18 smithi012 ceph-mon[23745]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:18.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:18 smithi174 ceph-mon[25682]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:19.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:22:19.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:22:19.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:19.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:22:19.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:22:19.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:20.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:20 smithi012 ceph-mon[23745]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:20.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:20 smithi012 ceph-mon[23745]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:22:20.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:20 smithi174 ceph-mon[25682]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:20.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:20 smithi174 ceph-mon[25682]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:22:22.015 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:21 smithi012 ceph-mon[23745]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:22.016 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:21 smithi174 ceph-mon[25682]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:21 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:23.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:22 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3618583014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d43fcc9-fb99-42f5-8e11-5f0ffe48d26c"}]: dispatch 2024-09-20T18:22:23.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:22 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3618583014' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d43fcc9-fb99-42f5-8e11-5f0ffe48d26c"}]': finished 2024-09-20T18:22:23.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:22 smithi012 ceph-mon[23745]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T18:22:23.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:22 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:23.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:22 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3618583014' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0d43fcc9-fb99-42f5-8e11-5f0ffe48d26c"}]: dispatch 2024-09-20T18:22:23.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:22 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3618583014' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0d43fcc9-fb99-42f5-8e11-5f0ffe48d26c"}]': finished 2024-09-20T18:22:23.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:22 smithi174 ceph-mon[25682]: osdmap e11: 2 total, 1 up, 2 in 2024-09-20T18:22:23.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:22 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:24.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:23 smithi012 ceph-mon[23745]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:24.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:23 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3734642996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:22:24.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:23 smithi174 ceph-mon[25682]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:24.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:23 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3734642996' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:22:26.119 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:25 smithi012 ceph-mon[23745]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:26.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:25 smithi174 ceph-mon[25682]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:28.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:27 smithi174 ceph-mon[25682]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:28.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:27 smithi012 ceph-mon[23745]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:29.896 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: Detected new or changed devices on smithi012 2024-09-20T18:22:29.896 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:29.896 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:22:29.896 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi012 to 12868M 2024-09-20T18:22:29.897 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:29.897 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:29.897 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:29.897 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:29.897 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:29.897 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:29 smithi012 ceph-mon[23745]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: Detected new or changed devices on smithi012 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi012 to 12868M 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:30.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:30.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:29 smithi174 ceph-mon[25682]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:31.681 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:31.682 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T18:22:31.682 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:31.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:31.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-20T18:22:31.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:32.595 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:32 smithi012 ceph-mon[23745]: Deploying daemon osd.1 on smithi012 2024-09-20T18:22:32.595 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:32 smithi012 ceph-mon[23745]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:32.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:32 smithi174 ceph-mon[25682]: Deploying daemon osd.1 on smithi012 2024-09-20T18:22:32.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:32 smithi174 ceph-mon[25682]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:34.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:34 smithi012 ceph-mon[23745]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:34.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:34 smithi174 ceph-mon[25682]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:36 smithi012 ceph-mon[23745]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:36.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:36 smithi174 ceph-mon[25682]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:36.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:36.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:38.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:38 smithi012 ceph-mon[23745]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:38.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:38.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:38.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:38.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:38.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:38 smithi174 ceph-mon[25682]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:38.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:38.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:38.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:38.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:40.458 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:40 smithi012 ceph-mon[23745]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:40.458 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:40 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:40.764 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 1 on host 'smithi012' 2024-09-20T18:22:40.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:40 smithi174 ceph-mon[25682]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:40.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:40 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:41.376 DEBUG:teuthology.orchestra.run.smithi012:osd.1> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.1.service 2024-09-20T18:22:41.379 INFO:tasks.cephadm:Deploying osd.2 on smithi012 with /dev/vg_nvme/lv_2... 2024-09-20T18:22:41.379 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T18:22:41.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:41 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:41.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:41 smithi012 ceph-mon[23745]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T18:22:41.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:41 smithi012 ceph-mon[23745]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:42.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:41 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:42.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:41 smithi174 ceph-mon[25682]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-20T18:22:42.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:41 smithi174 ceph-mon[25682]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:42.900 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:42 smithi012 ceph-mon[23745]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T18:22:42.900 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:42 smithi012 ceph-mon[23745]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T18:22:42.900 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:42.900 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:42 smithi012 ceph-mon[23745]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:22:42.901 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:22:42 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[38281]: 2024-09-20T18:22:42.762+0000 7fec1a3ee700 -1 osd.1 0 waiting for initial osdmap 2024-09-20T18:22:42.901 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:22:42 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[38281]: 2024-09-20T18:22:42.766+0000 7fec12d81700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:22:43.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:42 smithi174 ceph-mon[25682]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T18:22:43.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:42 smithi174 ceph-mon[25682]: osdmap e12: 2 total, 1 up, 2 in 2024-09-20T18:22:43.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:43.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:42 smithi174 ceph-mon[25682]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:22:43.920 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:43 smithi012 ceph-mon[23745]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:22:43.920 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:43 smithi012 ceph-mon[23745]: osdmap e13: 2 total, 1 up, 2 in 2024-09-20T18:22:43.920 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:43 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:43.920 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:43 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:43.920 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:43 smithi012 ceph-mon[23745]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:43.920 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:43 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:43 smithi174 ceph-mon[25682]: from='osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:22:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:43 smithi174 ceph-mon[25682]: osdmap e13: 2 total, 1 up, 2 in 2024-09-20T18:22:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:43 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:43 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:43 smithi174 ceph-mon[25682]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-20T18:22:44.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:43 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:45.060 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:44 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:22:45.060 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:44 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:22:45.060 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:44 smithi012 ceph-mon[23745]: osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944] boot 2024-09-20T18:22:45.060 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:44 smithi012 ceph-mon[23745]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T18:22:45.060 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:44 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:45.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:44 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:22:45.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:44 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:22:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:44 smithi174 ceph-mon[25682]: osd.1 [v2:172.21.15.12:6810/4097366944,v1:172.21.15.12:6811/4097366944] boot 2024-09-20T18:22:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:44 smithi174 ceph-mon[25682]: osdmap e14: 2 total, 2 up, 2 in 2024-09-20T18:22:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:44 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:22:45.349 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:22:45.368 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi012:vg_nvme/lv_2 2024-09-20T18:22:46.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:45 smithi012 ceph-mon[23745]: osdmap e15: 2 total, 2 up, 2 in 2024-09-20T18:22:46.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:45 smithi012 ceph-mon[23745]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:46.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:45 smithi174 ceph-mon[25682]: osdmap e15: 2 total, 2 up, 2 in 2024-09-20T18:22:46.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:45 smithi174 ceph-mon[25682]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:48.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:47 smithi012 ceph-mon[23745]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:48.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:47 smithi012 ceph-mon[23745]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:22:48.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:22:48.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:22:48.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:48.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:48.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:47 smithi174 ceph-mon[25682]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:48.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:47 smithi174 ceph-mon[25682]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:22:48.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:22:48.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:22:48.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:48.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:50.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:49 smithi012 ceph-mon[23745]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:50.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:49 smithi174 ceph-mon[25682]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:51.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:50 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/395526367' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "36737f27-549f-4fce-98ba-c1e03dd4e84a"}]: dispatch 2024-09-20T18:22:51.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:50 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/395526367' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "36737f27-549f-4fce-98ba-c1e03dd4e84a"}]': finished 2024-09-20T18:22:51.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:50 smithi174 ceph-mon[25682]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T18:22:51.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:50 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:22:51.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:50 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/395526367' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "36737f27-549f-4fce-98ba-c1e03dd4e84a"}]: dispatch 2024-09-20T18:22:51.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:50 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/395526367' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "36737f27-549f-4fce-98ba-c1e03dd4e84a"}]': finished 2024-09-20T18:22:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:50 smithi012 ceph-mon[23745]: osdmap e16: 3 total, 2 up, 3 in 2024-09-20T18:22:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:50 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:22:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:51 smithi174 ceph-mon[25682]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:51 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/209138046' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:22:52.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:51 smithi012 ceph-mon[23745]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:52.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:51 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/209138046' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:22:54.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:53 smithi174 ceph-mon[25682]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:54.336 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:53 smithi012 ceph-mon[23745]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:56.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:55 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:56.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:55 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:22:57.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:56 smithi174 ceph-mon[25682]: Detected new or changed devices on smithi012 2024-09-20T18:22:57.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:56 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi012 to 6434M 2024-09-20T18:22:57.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:56 smithi012 ceph-mon[23745]: Detected new or changed devices on smithi012 2024-09-20T18:22:57.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:56 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi012 to 6434M 2024-09-20T18:22:58.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:57 smithi174 ceph-mon[25682]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:58.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T18:22:58.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:58.300 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:57 smithi012 ceph-mon[23745]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:22:58.300 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-20T18:22:58.300 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:22:59.162 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:59 smithi012 ceph-mon[23745]: Deploying daemon osd.2 on smithi012 2024-09-20T18:22:59.162 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:22:59.162 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:22:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:22:59.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:59 smithi174 ceph-mon[25682]: Deploying daemon osd.2 on smithi012 2024-09-20T18:22:59.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:22:59.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:22:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:23:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:00 smithi174 ceph-mon[25682]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:00.398 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:00 smithi012 ceph-mon[23745]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:01.905 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:01 smithi012 ceph-mon[23745]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:02.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:01 smithi174 ceph-mon[25682]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:04.849 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:04 smithi012 ceph-mon[23745]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:04.849 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:04.849 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:05.017 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:04 smithi174 ceph-mon[25682]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:05.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:04 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:05.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:04 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:06.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:06 smithi012 ceph-mon[23745]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:06.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:06 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:06.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:06 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:06.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:06 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:06.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:06 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:06 smithi174 ceph-mon[25682]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:06 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:06 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:06 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:06 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:08.336 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:08 smithi012 ceph-mon[23745]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:08.367 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 2 on host 'smithi012' 2024-09-20T18:23:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:08 smithi174 ceph-mon[25682]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:08.964 DEBUG:teuthology.orchestra.run.smithi012:osd.2> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.2.service 2024-09-20T18:23:08.967 INFO:tasks.cephadm:Deploying osd.3 on smithi012 with /dev/vg_nvme/lv_1... 2024-09-20T18:23:08.967 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T18:23:09.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:09 smithi012 ceph-mon[23745]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T18:23:09.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:09.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:09 smithi174 ceph-mon[25682]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-20T18:23:09.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:09 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:10.450 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:10 smithi012 ceph-mon[23745]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:10.450 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:10 smithi012 ceph-mon[23745]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T18:23:10.450 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:10 smithi012 ceph-mon[23745]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T18:23:10.450 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:10 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:10.450 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:10 smithi012 ceph-mon[23745]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:23:10.450 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:10 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[41946]: 2024-09-20T18:23:10.106+0000 7f7b47753700 -1 osd.2 0 waiting for initial osdmap 2024-09-20T18:23:10.451 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:10 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[41946]: 2024-09-20T18:23:10.112+0000 7f7b400e6700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:23:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:10 smithi174 ceph-mon[25682]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:10 smithi174 ceph-mon[25682]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T18:23:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:10 smithi174 ceph-mon[25682]: osdmap e17: 3 total, 2 up, 3 in 2024-09-20T18:23:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:10 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:10 smithi174 ceph-mon[25682]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:23:11.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:11 smithi012 ceph-mon[23745]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:23:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:11 smithi012 ceph-mon[23745]: osdmap e18: 3 total, 2 up, 3 in 2024-09-20T18:23:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:11.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:11 smithi174 ceph-mon[25682]: from='osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:23:11.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:11 smithi174 ceph-mon[25682]: osdmap e18: 3 total, 2 up, 3 in 2024-09-20T18:23:11.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:11.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:12.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:12 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:23:12.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:12 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:23:12.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:12 smithi012 ceph-mon[23745]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:12.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:12.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:12 smithi012 ceph-mon[23745]: osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993] boot 2024-09-20T18:23:12.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:12 smithi012 ceph-mon[23745]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T18:23:12.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:12.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:12 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:23:12.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:12 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:23:12.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:12 smithi174 ceph-mon[25682]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-20T18:23:12.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:12.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:12 smithi174 ceph-mon[25682]: osd.2 [v2:172.21.15.12:6818/131798993,v1:172.21.15.12:6819/131798993] boot 2024-09-20T18:23:12.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:12 smithi174 ceph-mon[25682]: osdmap e19: 3 total, 3 up, 3 in 2024-09-20T18:23:12.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:23:12.685 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:23:12.705 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi012:vg_nvme/lv_1 2024-09-20T18:23:13.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:13 smithi012 ceph-mon[23745]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T18:23:13.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:13 smithi174 ceph-mon[25682]: osdmap e20: 3 total, 3 up, 3 in 2024-09-20T18:23:14.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:14 smithi012 ceph-mon[23745]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:14.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T18:23:14.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T18:23:14.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:14 smithi012 ceph-mon[23745]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T18:23:14.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:14 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T18:23:14.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:14 smithi174 ceph-mon[25682]: pgmap v75: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:14.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-20T18:23:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-20T18:23:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:14 smithi174 ceph-mon[25682]: osdmap e21: 3 total, 3 up, 3 in 2024-09-20T18:23:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:14 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-20T18:23:15.577 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:15 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T18:23:15.578 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:15 smithi012 ceph-mon[23745]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T18:23:15.578 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:15 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:15 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-20T18:23:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:15 smithi174 ceph-mon[25682]: osdmap e22: 3 total, 3 up, 3 in 2024-09-20T18:23:15.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:15 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:15.903 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:23:15 smithi012 sudo[43513]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T18:23:15.903 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:23:15 smithi012 sudo[43513]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:15.903 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:23:15 smithi012 sudo[43513]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:16.472 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:23:16 smithi012 sudo[43532]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T18:23:16.472 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:23:16 smithi012 sudo[43532]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:16.472 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:23:16 smithi012 sudo[43532]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:16 smithi174 ceph-mon[25682]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:16 smithi174 ceph-mon[25682]: osdmap e23: 3 total, 3 up, 3 in 2024-09-20T18:23:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:16 smithi174 ceph-mon[25682]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:23:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:16 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:23:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:16 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:23:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:16 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:16 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:16.798 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:23:16 smithi012 sudo[43564]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T18:23:16.798 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:23:16 smithi012 sudo[43564]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:16.798 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:23:16 smithi012 sudo[43564]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:16.799 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:16 smithi012 ceph-mon[23745]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:16.799 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:16 smithi012 ceph-mon[23745]: osdmap e23: 3 total, 3 up, 3 in 2024-09-20T18:23:16.799 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:16 smithi012 ceph-mon[23745]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi012:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:23:16.799 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:16 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:23:16.799 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:16 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:23:16.799 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:16 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:16.799 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:16 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:17.464 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:23:17 smithi012 sudo[43593]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T18:23:17.464 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:23:17 smithi012 sudo[43593]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:17.464 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:23:17 smithi012 sudo[43593]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:17.722 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:17 smithi012 sudo[43636]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T18:23:17.722 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:17 smithi012 sudo[43636]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:17.722 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:17 smithi012 sudo[43636]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:18.181 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43664]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T18:23:18.181 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43664]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:18.181 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43664]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:18.908 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43672]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T18:23:18.908 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43672]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:18.908 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43672]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:19.172 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43697]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T18:23:19.172 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43697]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:19.172 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 sudo[43697]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:19.172 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 ceph-mon[23745]: pgmap v80: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:19.172 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 ceph-mon[23745]: mgrmap e19: smithi012.xkxxrd(active, since 2m), standbys: smithi174.nxkptu 2024-09-20T18:23:19.172 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:18 smithi012 ceph-mon[23745]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T18:23:19.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:18 smithi174 ceph-mon[25682]: pgmap v80: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:19.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:18 smithi174 ceph-mon[25682]: mgrmap e19: smithi012.xkxxrd(active, since 2m), standbys: smithi174.nxkptu 2024-09-20T18:23:19.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:18 smithi174 ceph-mon[25682]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T18:23:19.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 sudo[28006]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-20T18:23:19.247 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 sudo[28006]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:19.248 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 sudo[28006]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:19.517 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 sudo[28009]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-20T18:23:19.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 sudo[28009]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-20T18:23:19.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 sudo[28009]: pam_unix(sudo:session): session closed for user root 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: pgmap v81: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi012 to 4289M 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:19 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: pgmap v81: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi012 to 4289M 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:20.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:19 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:21.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:20 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3321818369' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dde6071d-ec62-461f-8ce4-d9d43c296320"}]: dispatch 2024-09-20T18:23:21.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:20 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3321818369' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dde6071d-ec62-461f-8ce4-d9d43c296320"}]': finished 2024-09-20T18:23:21.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:20 smithi174 ceph-mon[25682]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T18:23:21.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:20 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:21.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:20 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/4141663505' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:23:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:20 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3321818369' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dde6071d-ec62-461f-8ce4-d9d43c296320"}]: dispatch 2024-09-20T18:23:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:20 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3321818369' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dde6071d-ec62-461f-8ce4-d9d43c296320"}]': finished 2024-09-20T18:23:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:20 smithi012 ceph-mon[23745]: osdmap e24: 4 total, 3 up, 4 in 2024-09-20T18:23:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:20 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:20 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/4141663505' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:23:22.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:21 smithi174 ceph-mon[25682]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:22.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:21 smithi012 ceph-mon[23745]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:24.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:23 smithi012 ceph-mon[23745]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:24.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:23 smithi174 ceph-mon[25682]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:26.023 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:25 smithi012 ceph-mon[23745]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:26.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:25 smithi174 ceph-mon[25682]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:27.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T18:23:27.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:27.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:27 smithi174 ceph-mon[25682]: Deploying daemon osd.3 on smithi012 2024-09-20T18:23:27.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-20T18:23:27.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:27.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:27 smithi012 ceph-mon[23745]: Deploying daemon osd.3 on smithi012 2024-09-20T18:23:28.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:28 smithi174 ceph-mon[25682]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:28.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:28 smithi012 ceph-mon[23745]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:30.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:30 smithi012 ceph-mon[23745]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:30 smithi174 ceph-mon[25682]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:31.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:31 smithi012 ceph-mon[23745]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:31.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:31.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:31.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:31.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:31 smithi174 ceph-mon[25682]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:34.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:33 smithi174 ceph-mon[25682]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:34.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:33 smithi012 ceph-mon[23745]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:35.202 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:34 smithi012 ceph-mon[23745]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T18:23:35.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:34 smithi174 ceph-mon[25682]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-20T18:23:35.470 INFO:teuthology.orchestra.run.smithi012.stdout:Created osd(s) 3 on host 'smithi012' 2024-09-20T18:23:36.097 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:35 smithi012 ceph-mon[23745]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:36.097 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:35 smithi012 ceph-mon[23745]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T18:23:36.097 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:35 smithi012 ceph-mon[23745]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T18:23:36.097 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:35 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:36.097 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:35 smithi012 ceph-mon[23745]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:23:36.097 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:35 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:36.121 DEBUG:teuthology.orchestra.run.smithi012:osd.3> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.3.service 2024-09-20T18:23:36.124 INFO:tasks.cephadm:Deploying osd.4 on smithi174 with /dev/vg_nvme/lv_4... 2024-09-20T18:23:36.124 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-20T18:23:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:35 smithi174 ceph-mon[25682]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:35 smithi174 ceph-mon[25682]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T18:23:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:35 smithi174 ceph-mon[25682]: osdmap e25: 4 total, 3 up, 4 in 2024-09-20T18:23:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:35 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:35 smithi174 ceph-mon[25682]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]: dispatch 2024-09-20T18:23:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:35 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:37.262 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:36 smithi012 ceph-mon[23745]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:23:37.262 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:36 smithi012 ceph-mon[23745]: osdmap e26: 4 total, 3 up, 4 in 2024-09-20T18:23:37.262 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:37.262 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:37.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:36 smithi174 ceph-mon[25682]: from='osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]}]': finished 2024-09-20T18:23:37.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:36 smithi174 ceph-mon[25682]: osdmap e26: 4 total, 3 up, 4 in 2024-09-20T18:23:37.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:37.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:38.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:37 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:23:38.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:37 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:23:38.204 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:37 smithi174 ceph-mon[25682]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:38.205 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:37 smithi174 ceph-mon[25682]: osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100] boot 2024-09-20T18:23:38.205 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:37 smithi174 ceph-mon[25682]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T18:23:38.205 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:38.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:37 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:23:38.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:37 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:23:38.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:37 smithi012 ceph-mon[23745]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-20T18:23:38.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:37 smithi012 ceph-mon[23745]: osd.3 [v2:172.21.15.12:6826/3312213100,v1:172.21.15.12:6827/3312213100] boot 2024-09-20T18:23:38.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:37 smithi012 ceph-mon[23745]: osdmap e27: 4 total, 4 up, 4 in 2024-09-20T18:23:38.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:23:38.451 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:23:38.472 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi174:vg_nvme/lv_4 2024-09-20T18:23:39.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:39 smithi174 ceph-mon[25682]: osdmap e28: 4 total, 4 up, 4 in 2024-09-20T18:23:39.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:39 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:39.343 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:39 smithi012 ceph-mon[23745]: osdmap e28: 4 total, 4 up, 4 in 2024-09-20T18:23:39.343 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:39 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:40.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:40 smithi012 ceph-mon[23745]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:40.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:40 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:23:40.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:40 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:23:40.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:40 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:40.745 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:40 smithi174 ceph-mon[25682]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:40.745 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:40 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:23:40.745 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:40 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:23:40.745 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:40 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:41.536 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:41 smithi012 ceph-mon[23745]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:23:41.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:41 smithi174 ceph-mon[25682]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:23:42.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:42.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:42 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:42.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:42.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:42.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:42.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:42 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:43.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: Detected new or changed devices on smithi012 2024-09-20T18:23:43.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi012 to 3217M 2024-09-20T18:23:43.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/3125935573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41c03da2-744a-451b-95a2-ae934bba44cc"}]: dispatch 2024-09-20T18:23:43.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41c03da2-744a-451b-95a2-ae934bba44cc"}]: dispatch 2024-09-20T18:23:43.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "41c03da2-744a-451b-95a2-ae934bba44cc"}]': finished 2024-09-20T18:23:43.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T18:23:43.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:43.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:43 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/1565238756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: Detected new or changed devices on smithi012 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi012 to 3217M 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/3125935573' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41c03da2-744a-451b-95a2-ae934bba44cc"}]: dispatch 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "41c03da2-744a-451b-95a2-ae934bba44cc"}]: dispatch 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "41c03da2-744a-451b-95a2-ae934bba44cc"}]': finished 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: osdmap e29: 5 total, 4 up, 5 in 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:43.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:43 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/1565238756' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:23:44.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:44 smithi174 ceph-mon[25682]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:44.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:44 smithi012 ceph-mon[23745]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:46.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:46 smithi174 ceph-mon[25682]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:46.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:46 smithi012 ceph-mon[23745]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:48 smithi174 ceph-mon[25682]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:48 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T18:23:48.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:48 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:48.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:48 smithi012 ceph-mon[23745]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:48.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:48 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-20T18:23:48.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:48 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:49.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:49 smithi174 ceph-mon[25682]: Deploying daemon osd.4 on smithi174 2024-09-20T18:23:49.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:49 smithi012 ceph-mon[23745]: Deploying daemon osd.4 on smithi174 2024-09-20T18:23:50.933 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:50 smithi174 ceph-mon[25682]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:50.972 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:50 smithi012 ceph-mon[23745]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:52.017 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:51 smithi174 ceph-mon[25682]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:52.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:51 smithi012 ceph-mon[23745]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:54.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:54 smithi012 ceph-mon[23745]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:54.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:54 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:54.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:54 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:54.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:54 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:54.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:54 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:54.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:54 smithi174 ceph-mon[25682]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:54.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:54 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:54.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:54 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:54.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:54 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:54.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:54 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:56.362 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:56 smithi174 ceph-mon[25682]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:56.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:56 smithi012 ceph-mon[23745]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:56.765 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 4 on host 'smithi174' 2024-09-20T18:23:57.395 DEBUG:teuthology.orchestra.run.smithi174:osd.4> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.4.service 2024-09-20T18:23:57.398 INFO:tasks.cephadm:Deploying osd.5 on smithi174 with /dev/vg_nvme/lv_3... 2024-09-20T18:23:57.398 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='osd.4 [v2:172.21.15.174:6800/2768749008,v1:172.21.15.174:6801/2768749008]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:57.708 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='osd.4 [v2:172.21.15.174:6800/2768749008,v1:172.21.15.174:6801/2768749008]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:23:57.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:23:58.657 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-mon[25682]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:58.657 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-mon[25682]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T18:23:58.657 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-mon[25682]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T18:23:58.657 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:58.657 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-mon[25682]: from='osd.4 [v2:172.21.15.174:6800/2768749008,v1:172.21.15.174:6801/2768749008]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:23:58.657 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-mon[25682]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:23:58.658 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[29913]: 2024-09-20T18:23:58.370+0000 7fde45452700 -1 osd.4 0 waiting for initial osdmap 2024-09-20T18:23:58.658 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:23:58 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[29913]: 2024-09-20T18:23:58.374+0000 7fde3fde9700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:23:58.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:58 smithi012 ceph-mon[23745]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:23:58.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:58 smithi012 ceph-mon[23745]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T18:23:58.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:58 smithi012 ceph-mon[23745]: osdmap e30: 5 total, 4 up, 5 in 2024-09-20T18:23:58.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:58 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:58.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:58 smithi012 ceph-mon[23745]: from='osd.4 [v2:172.21.15.174:6800/2768749008,v1:172.21.15.174:6801/2768749008]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:23:58.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:58 smithi012 ceph-mon[23745]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:23:59.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:59 smithi174 ceph-mon[25682]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:23:59.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:59 smithi174 ceph-mon[25682]: osdmap e31: 5 total, 4 up, 5 in 2024-09-20T18:23:59.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:59.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:59.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:23:59.562 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:23:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:23:59.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:59 smithi012 ceph-mon[23745]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:23:59.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:59 smithi012 ceph-mon[23745]: osdmap e31: 5 total, 4 up, 5 in 2024-09-20T18:23:59.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:59.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:23:59.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:23:59.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:23:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:24:00.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:00 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:24:00.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:00 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:24:00.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:00 smithi012 ceph-mon[23745]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:24:00.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:00 smithi012 ceph-mon[23745]: osd.4 [v2:172.21.15.174:6800/2768749008,v1:172.21.15.174:6801/2768749008] boot 2024-09-20T18:24:00.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:00 smithi012 ceph-mon[23745]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T18:24:00.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:24:00.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:00 smithi012 ceph-mon[23745]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T18:24:00.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:00 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:24:00.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:00 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:24:00.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:00 smithi174 ceph-mon[25682]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-20T18:24:00.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:00 smithi174 ceph-mon[25682]: osd.4 [v2:172.21.15.174:6800/2768749008,v1:172.21.15.174:6801/2768749008] boot 2024-09-20T18:24:00.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:00 smithi174 ceph-mon[25682]: osdmap e32: 5 total, 5 up, 5 in 2024-09-20T18:24:00.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:24:00.759 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:00 smithi174 ceph-mon[25682]: osdmap e33: 5 total, 5 up, 5 in 2024-09-20T18:24:00.764 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:24:00.786 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi174:vg_nvme/lv_3 2024-09-20T18:24:02.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:02 smithi012 ceph-mon[23745]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:02.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:02 smithi012 ceph-mon[23745]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T18:24:02.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:02 smithi174 ceph-mon[25682]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:02.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:02 smithi174 ceph-mon[25682]: osdmap e34: 5 total, 5 up, 5 in 2024-09-20T18:24:03.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:03 smithi012 ceph-mon[23745]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T18:24:03.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:03.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:03.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:03.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:03.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:03 smithi012 ceph-mon[23745]: osdmap e36: 5 total, 5 up, 5 in 2024-09-20T18:24:03.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:03 smithi174 ceph-mon[25682]: osdmap e35: 5 total, 5 up, 5 in 2024-09-20T18:24:03.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:03.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:03.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:03.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:03.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:03 smithi174 ceph-mon[25682]: osdmap e36: 5 total, 5 up, 5 in 2024-09-20T18:24:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:04 smithi012 ceph-mon[23745]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:04 smithi012 ceph-mon[23745]: Detected new or changed devices on smithi174 2024-09-20T18:24:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:04 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi174 to 15936M 2024-09-20T18:24:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:04 smithi012 ceph-mon[23745]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:24:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:24:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:24:04.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:04.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:04 smithi174 ceph-mon[25682]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:04.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:04 smithi174 ceph-mon[25682]: Detected new or changed devices on smithi174 2024-09-20T18:24:04.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:04 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi174 to 15936M 2024-09-20T18:24:04.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:04 smithi174 ceph-mon[25682]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:24:04.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:04 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:24:04.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:04 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:24:04.738 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:04 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:06.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:06 smithi012 ceph-mon[23745]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-09-20T18:24:06.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:06 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/1829689491' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3375f2c1-371d-4256-b5fd-1bf87b2aeea2"}]: dispatch 2024-09-20T18:24:06.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:06 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3375f2c1-371d-4256-b5fd-1bf87b2aeea2"}]: dispatch 2024-09-20T18:24:06.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:06 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3375f2c1-371d-4256-b5fd-1bf87b2aeea2"}]': finished 2024-09-20T18:24:06.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:06 smithi012 ceph-mon[23745]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T18:24:06.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:06 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:06.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:06 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/1317090922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:24:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:06 smithi174 ceph-mon[25682]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-09-20T18:24:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:06 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/1829689491' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3375f2c1-371d-4256-b5fd-1bf87b2aeea2"}]: dispatch 2024-09-20T18:24:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:06 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3375f2c1-371d-4256-b5fd-1bf87b2aeea2"}]: dispatch 2024-09-20T18:24:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:06 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3375f2c1-371d-4256-b5fd-1bf87b2aeea2"}]': finished 2024-09-20T18:24:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:06 smithi174 ceph-mon[25682]: osdmap e37: 6 total, 5 up, 6 in 2024-09-20T18:24:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:06 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:06.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:06 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/1317090922' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:24:08.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:08 smithi012 ceph-mon[23745]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-20T18:24:08.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:08 smithi174 ceph-mon[25682]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-09-20T18:24:10.605 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:10 smithi174 ceph-mon[25682]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T18:24:10.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:10 smithi012 ceph-mon[23745]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 79 KiB/s, 0 objects/s recovering 2024-09-20T18:24:12.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:11 smithi174 ceph-mon[25682]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T18:24:12.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T18:24:12.189 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:12.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:11 smithi012 ceph-mon[23745]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T18:24:12.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-20T18:24:12.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:13.194 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:12 smithi174 ceph-mon[25682]: Deploying daemon osd.5 on smithi174 2024-09-20T18:24:13.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:12 smithi012 ceph-mon[23745]: Deploying daemon osd.5 on smithi174 2024-09-20T18:24:14.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:13 smithi012 ceph-mon[23745]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-20T18:24:14.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:13 smithi174 ceph-mon[25682]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-09-20T18:24:16.093 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:15 smithi174 ceph-mon[25682]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:16.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:15 smithi012 ceph-mon[23745]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:17.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:17 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:17.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:17 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:17.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:17 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:17.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:17 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:17.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:17 smithi012 ceph-mon[23745]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:18.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:17 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:18.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:17 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:18.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:17 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:18.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:17 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:18.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:17 smithi174 ceph-mon[25682]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:20.020 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 5 on host 'smithi174' 2024-09-20T18:24:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:19 smithi174 ceph-mon[25682]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:19 smithi174 ceph-mon[25682]: from='osd.5 [v2:172.21.15.174:6808/3359326262,v1:172.21.15.174:6809/3359326262]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T18:24:20.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:19 smithi174 ceph-mon[25682]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T18:24:20.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:19 smithi012 ceph-mon[23745]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:20.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:19 smithi012 ceph-mon[23745]: from='osd.5 [v2:172.21.15.174:6808/3359326262,v1:172.21.15.174:6809/3359326262]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T18:24:20.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:19 smithi012 ceph-mon[23745]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-20T18:24:20.881 DEBUG:teuthology.orchestra.run.smithi174:osd.5> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.5.service 2024-09-20T18:24:20.883 INFO:tasks.cephadm:Deploying osd.6 on smithi174 with /dev/vg_nvme/lv_2... 2024-09-20T18:24:20.883 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-20T18:24:21.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-mon[25682]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T18:24:21.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-mon[25682]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T18:24:21.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:21.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-mon[25682]: from='osd.5 [v2:172.21.15.174:6808/3359326262,v1:172.21.15.174:6809/3359326262]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:21.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-mon[25682]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:21.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:21.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:21.191 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[32860]: 2024-09-20T18:24:21.013+0000 7fe1e72dd700 -1 osd.5 0 waiting for initial osdmap 2024-09-20T18:24:21.191 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:24:21 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[32860]: 2024-09-20T18:24:21.020+0000 7fe1e0c72700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:24:21.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:21 smithi012 ceph-mon[23745]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T18:24:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:21 smithi012 ceph-mon[23745]: osdmap e38: 6 total, 5 up, 6 in 2024-09-20T18:24:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:21 smithi012 ceph-mon[23745]: from='osd.5 [v2:172.21.15.174:6808/3359326262,v1:172.21.15.174:6809/3359326262]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:21 smithi012 ceph-mon[23745]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:21.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:21 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:22.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:22.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:24:22.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: osdmap e39: 6 total, 5 up, 6 in 2024-09-20T18:24:22.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:22.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:22.267 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: osd.5 [v2:172.21.15.174:6808/3359326262,v1:172.21.15.174:6809/3359326262] boot 2024-09-20T18:24:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T18:24:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:22 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:22.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-20T18:24:22.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:24:22.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: osdmap e39: 6 total, 5 up, 6 in 2024-09-20T18:24:22.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:22.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:22.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: osd.5 [v2:172.21.15.174:6808/3359326262,v1:172.21.15.174:6809/3359326262] boot 2024-09-20T18:24:22.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: osdmap e40: 6 total, 6 up, 6 in 2024-09-20T18:24:22.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:22 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:24:23.119 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:23 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:24:23.119 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:23 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:24:23.119 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:23 smithi174 ceph-mon[25682]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T18:24:23.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:23 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:24:23.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:23 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:24:23.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:23 smithi012 ceph-mon[23745]: osdmap e41: 6 total, 6 up, 6 in 2024-09-20T18:24:24.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:24 smithi174 ceph-mon[25682]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:24.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:24 smithi174 ceph-mon[25682]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T18:24:24.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:24 smithi174 ceph-mon[25682]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T18:24:24.294 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:24:24.318 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi174:vg_nvme/lv_2 2024-09-20T18:24:24.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:24 smithi012 ceph-mon[23745]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:24.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:24 smithi012 ceph-mon[23745]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T18:24:24.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:24 smithi012 ceph-mon[23745]: osdmap e42: 6 total, 6 up, 6 in 2024-09-20T18:24:26.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:26 smithi174 ceph-mon[25682]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:26.168 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:26 smithi174 ceph-mon[25682]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T18:24:26.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:26 smithi012 ceph-mon[23745]: pgmap v133: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:26.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:26 smithi012 ceph-mon[23745]: osdmap e43: 6 total, 6 up, 6 in 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: osdmap e44: 6 total, 6 up, 6 in 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:27.180 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:27.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:27.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:27.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:27.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:27.181 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:27 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:27.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: osdmap e44: 6 total, 6 up, 6 in 2024-09-20T18:24:27.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:24:27.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:24:27.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:24:27.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:27.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:27.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:27.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:27.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:27.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:27.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:27.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:27.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:27.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:27 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:28.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:28 smithi174 ceph-mon[25682]: Detected new or changed devices on smithi174 2024-09-20T18:24:28.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:28 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi174 to 7968M 2024-09-20T18:24:28.319 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:28 smithi174 ceph-mon[25682]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:28.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:28 smithi012 ceph-mon[23745]: Detected new or changed devices on smithi174 2024-09-20T18:24:28.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:28 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi174 to 7968M 2024-09-20T18:24:28.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:28 smithi012 ceph-mon[23745]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:30.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-20T18:24:30.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:30.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T18:24:30.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: Cluster is now healthy 2024-09-20T18:24:30.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/2969492729' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "abf17c05-829d-4c3e-abb5-a1ffa2bdd583"}]: dispatch 2024-09-20T18:24:30.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "abf17c05-829d-4c3e-abb5-a1ffa2bdd583"}]: dispatch 2024-09-20T18:24:30.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "abf17c05-829d-4c3e-abb5-a1ffa2bdd583"}]': finished 2024-09-20T18:24:30.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T18:24:30.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:30.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:29 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/948673496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:24:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-09-20T18:24:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T18:24:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: Cluster is now healthy 2024-09-20T18:24:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/2969492729' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "abf17c05-829d-4c3e-abb5-a1ffa2bdd583"}]: dispatch 2024-09-20T18:24:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "abf17c05-829d-4c3e-abb5-a1ffa2bdd583"}]: dispatch 2024-09-20T18:24:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "abf17c05-829d-4c3e-abb5-a1ffa2bdd583"}]': finished 2024-09-20T18:24:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: osdmap e45: 7 total, 6 up, 7 in 2024-09-20T18:24:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:29 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/948673496' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:24:32.176 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:31 smithi174 ceph-mon[25682]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T18:24:32.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:31 smithi012 ceph-mon[23745]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T18:24:34.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:33 smithi174 ceph-mon[25682]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T18:24:34.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:33 smithi012 ceph-mon[23745]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-20T18:24:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:36 smithi174 ceph-mon[25682]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-20T18:24:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T18:24:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:36.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:36 smithi174 ceph-mon[25682]: Deploying daemon osd.6 on smithi174 2024-09-20T18:24:36.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:35 smithi012 ceph-mon[23745]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-20T18:24:36.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:35 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-20T18:24:36.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:35 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:36.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:35 smithi012 ceph-mon[23745]: Deploying daemon osd.6 on smithi174 2024-09-20T18:24:38.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:38 smithi174 ceph-mon[25682]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T18:24:38.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:38 smithi012 ceph-mon[23745]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T18:24:40.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:40 smithi174 ceph-mon[25682]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:40.458 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:40 smithi012 ceph-mon[23745]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:41.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:41 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:41.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:41 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:41.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:41 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:41.317 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:41 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:41.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:41 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:41.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:41 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:41.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:41 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:41.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:41 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:42.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:42 smithi174 ceph-mon[25682]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:42.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:42 smithi012 ceph-mon[23745]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:43.282 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 6 on host 'smithi174' 2024-09-20T18:24:43.895 DEBUG:teuthology.orchestra.run.smithi174:osd.6> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.6.service 2024-09-20T18:24:43.897 INFO:tasks.cephadm:Deploying osd.7 on smithi174 with /dev/vg_nvme/lv_1... 2024-09-20T18:24:43.898 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-20T18:24:44.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:44 smithi012 ceph-mon[23745]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:44.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:44 smithi012 ceph-mon[23745]: from='osd.6 [v2:172.21.15.174:6816/1588901062,v1:172.21.15.174:6817/1588901062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T18:24:44.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:44 smithi012 ceph-mon[23745]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T18:24:44.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:44 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:44.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:44 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:44.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:44 smithi174 ceph-mon[25682]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:44.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:44 smithi174 ceph-mon[25682]: from='osd.6 [v2:172.21.15.174:6816/1588901062,v1:172.21.15.174:6817/1588901062]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T18:24:44.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:44 smithi174 ceph-mon[25682]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-20T18:24:44.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:44 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:44.488 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:44 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:45 smithi174 ceph-mon[25682]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T18:24:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:45 smithi174 ceph-mon[25682]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T18:24:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:45 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:45 smithi174 ceph-mon[25682]: from='osd.6 [v2:172.21.15.174:6816/1588901062,v1:172.21.15.174:6817/1588901062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:45.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:45 smithi174 ceph-mon[25682]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:45.268 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:24:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[35875]: 2024-09-20T18:24:45.131+0000 7f65cca82700 -1 osd.6 0 waiting for initial osdmap 2024-09-20T18:24:45.269 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:24:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[35875]: 2024-09-20T18:24:45.137+0000 7f65c7419700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:24:45.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:45 smithi012 ceph-mon[23745]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T18:24:45.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:45 smithi012 ceph-mon[23745]: osdmap e46: 7 total, 6 up, 7 in 2024-09-20T18:24:45.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:45 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:45.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:45 smithi012 ceph-mon[23745]: from='osd.6 [v2:172.21.15.174:6816/1588901062,v1:172.21.15.174:6817/1588901062]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:45.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:45 smithi012 ceph-mon[23745]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:24:46.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:46 smithi174 ceph-mon[25682]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:46.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:46 smithi174 ceph-mon[25682]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:24:46.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:46 smithi174 ceph-mon[25682]: osdmap e47: 7 total, 6 up, 7 in 2024-09-20T18:24:46.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:46.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:46.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:46 smithi012 ceph-mon[23745]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:46 smithi012 ceph-mon[23745]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:24:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:46 smithi012 ceph-mon[23745]: osdmap e47: 7 total, 6 up, 7 in 2024-09-20T18:24:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:47.178 INFO:teuthology.orchestra.run.smithi174.stdout: 2024-09-20T18:24:47.201 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch daemon add osd smithi174:vg_nvme/lv_1 2024-09-20T18:24:47.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:47 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:24:47.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:47 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:24:47.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:47.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:47 smithi012 ceph-mon[23745]: osd.6 [v2:172.21.15.174:6816/1588901062,v1:172.21.15.174:6817/1588901062] boot 2024-09-20T18:24:47.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:47 smithi012 ceph-mon[23745]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T18:24:47.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:47 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:47.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:47 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:24:47.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:47 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:24:47.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:47.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:47 smithi174 ceph-mon[25682]: osd.6 [v2:172.21.15.174:6816/1588901062,v1:172.21.15.174:6817/1588901062] boot 2024-09-20T18:24:47.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:47 smithi174 ceph-mon[25682]: osdmap e48: 7 total, 7 up, 7 in 2024-09-20T18:24:47.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:47 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:24:48.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:48 smithi174 ceph-mon[25682]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:48.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:48 smithi174 ceph-mon[25682]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T18:24:48.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:48 smithi012 ceph-mon[23745]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-20T18:24:48.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:48 smithi012 ceph-mon[23745]: osdmap e49: 7 total, 7 up, 7 in 2024-09-20T18:24:49.436 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:49 smithi174 ceph-mon[25682]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T18:24:49.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:49 smithi012 ceph-mon[23745]: osdmap e50: 7 total, 7 up, 7 in 2024-09-20T18:24:50.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:50 smithi174 ceph-mon[25682]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:24:50.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:50 smithi174 ceph-mon[25682]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T18:24:50.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:50 smithi012 ceph-mon[23745]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:24:50.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:50 smithi012 ceph-mon[23745]: osdmap e51: 7 total, 7 up, 7 in 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: osdmap e52: 7 total, 7 up, 7 in 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: Detected new or changed devices on smithi174 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi174 to 5312M 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.471 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:24:51.472 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:24:51.472 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: osdmap e52: 7 total, 7 up, 7 in 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: Detected new or changed devices on smithi174 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi174 to 5312M 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:51.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:24:51.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:24:51.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-20T18:24:51.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-20T18:24:51.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:24:52.468 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:52 smithi174 ceph-mon[25682]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:24:52.468 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:52 smithi174 ceph-mon[25682]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:24:52.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:52 smithi012 ceph-mon[23745]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi174:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:24:52.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:52 smithi012 ceph-mon[23745]: pgmap v156: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:24:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:53 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/1946965342' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11ca6b60-4c5f-452d-ac4d-b6be024590a6"}]: dispatch 2024-09-20T18:24:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:53 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11ca6b60-4c5f-452d-ac4d-b6be024590a6"}]: dispatch 2024-09-20T18:24:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:53 smithi012 ceph-mon[23745]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11ca6b60-4c5f-452d-ac4d-b6be024590a6"}]': finished 2024-09-20T18:24:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:53 smithi012 ceph-mon[23745]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T18:24:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:24:53.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:53 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/1946965342' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11ca6b60-4c5f-452d-ac4d-b6be024590a6"}]: dispatch 2024-09-20T18:24:53.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:53 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11ca6b60-4c5f-452d-ac4d-b6be024590a6"}]: dispatch 2024-09-20T18:24:53.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:53 smithi174 ceph-mon[25682]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11ca6b60-4c5f-452d-ac4d-b6be024590a6"}]': finished 2024-09-20T18:24:53.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:53 smithi174 ceph-mon[25682]: osdmap e53: 8 total, 7 up, 8 in 2024-09-20T18:24:53.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:24:54.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:54 smithi012 ceph-mon[23745]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-20T18:24:54.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:54 smithi012 ceph-mon[23745]: from='client.? 172.21.15.174:0/2276624073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:24:54.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:54 smithi174 ceph-mon[25682]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-20T18:24:54.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:54 smithi174 ceph-mon[25682]: from='client.? 172.21.15.174:0/2276624073' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-20T18:24:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:56 smithi174 ceph-mon[25682]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T18:24:56.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:56 smithi012 ceph-mon[23745]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T18:24:58.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:58 smithi012 ceph-mon[23745]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-20T18:24:58.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:58 smithi174 ceph-mon[25682]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-20T18:24:59.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:24:59.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:24:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:24:59.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:24:59.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:24:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:25:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:00 smithi174 ceph-mon[25682]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-20T18:25:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T18:25:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:25:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:00 smithi174 ceph-mon[25682]: Deploying daemon osd.7 on smithi174 2024-09-20T18:25:00.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:00 smithi012 ceph-mon[23745]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-20T18:25:00.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-20T18:25:00.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:25:00.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:00 smithi012 ceph-mon[23745]: Deploying daemon osd.7 on smithi174 2024-09-20T18:25:02.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:01 smithi174 ceph-mon[25682]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T18:25:02.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:01 smithi012 ceph-mon[23745]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-20T18:25:04.238 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:04 smithi174 ceph-mon[25682]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:04.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:04 smithi012 ceph-mon[23745]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:05.722 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:05 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:05.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:05 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:25:05.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:05 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:25:05.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:05 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:25:05.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:05 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:05.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:05 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:25:05.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:05 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:25:05.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:05 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:25:06.405 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:06 smithi174 ceph-mon[25682]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:06.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:06 smithi012 ceph-mon[23745]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:07.650 INFO:teuthology.orchestra.run.smithi174.stdout:Created osd(s) 7 on host 'smithi174' 2024-09-20T18:25:08.270 DEBUG:teuthology.orchestra.run.smithi174:osd.7> sudo journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.7.service 2024-09-20T18:25:08.273 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-20T18:25:08.273 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd stat -f json 2024-09-20T18:25:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:08 smithi174 ceph-mon[25682]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:08 smithi174 ceph-mon[25682]: from='osd.7 [v2:172.21.15.174:6824/3839005316,v1:172.21.15.174:6825/3839005316]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T18:25:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:08 smithi174 ceph-mon[25682]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T18:25:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:08 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:08.592 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:08 smithi012 ceph-mon[23745]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:08.592 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:08 smithi012 ceph-mon[23745]: from='osd.7 [v2:172.21.15.174:6824/3839005316,v1:172.21.15.174:6825/3839005316]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T18:25:08.592 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:08 smithi012 ceph-mon[23745]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-20T18:25:08.592 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:08 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:09.716 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:09 smithi012 ceph-mon[23745]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T18:25:09.716 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:09 smithi012 ceph-mon[23745]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T18:25:09.716 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:09.716 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:09 smithi012 ceph-mon[23745]: from='osd.7 [v2:172.21.15.174:6824/3839005316,v1:172.21.15.174:6825/3839005316]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:25:09.716 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:09 smithi012 ceph-mon[23745]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:25:09.716 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:09.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-mon[25682]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T18:25:09.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-mon[25682]: osdmap e54: 8 total, 7 up, 8 in 2024-09-20T18:25:09.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:09.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-mon[25682]: from='osd.7 [v2:172.21.15.174:6824/3839005316,v1:172.21.15.174:6825/3839005316]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:25:09.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-mon[25682]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]: dispatch 2024-09-20T18:25:09.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:09.769 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[38851]: 2024-09-20T18:25:09.366+0000 7f7eee28a700 -1 osd.7 0 waiting for initial osdmap 2024-09-20T18:25:09.769 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:25:09 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[38851]: 2024-09-20T18:25:09.374+0000 7f7eea424700 -1 osd.7 55 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:25:10.401 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:25:10.700 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:10 smithi012 ceph-mon[23745]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:10.700 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:10 smithi012 ceph-mon[23745]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:25:10.700 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:10 smithi012 ceph-mon[23745]: osdmap e55: 8 total, 7 up, 8 in 2024-09-20T18:25:10.700 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:10 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:10.701 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:10 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:10.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:10 smithi174 ceph-mon[25682]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:10.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:10 smithi174 ceph-mon[25682]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]}]': finished 2024-09-20T18:25:10.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:10 smithi174 ceph-mon[25682]: osdmap e55: 8 total, 7 up, 8 in 2024-09-20T18:25:10.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:10 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:10.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:10 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:11.214 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1726856710,"num_in_osds":8,"osd_in_since":1726856692,"num_remapped_pgs":1} 2024-09-20T18:25:11.215 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T18:25:11.438 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:11.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: purged_snaps scrub starts 2024-09-20T18:25:11.472 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: purged_snaps scrub ok 2024-09-20T18:25:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: osd.7 [v2:172.21.15.174:6824/3839005316,v1:172.21.15.174:6825/3839005316] boot 2024-09-20T18:25:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T18:25:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1250289087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T18:25:11.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:11 smithi012 ceph-mon[23745]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T18:25:11.701 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: purged_snaps scrub starts 2024-09-20T18:25:11.701 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: purged_snaps scrub ok 2024-09-20T18:25:11.701 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:11.701 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: osd.7 [v2:172.21.15.174:6824/3839005316,v1:172.21.15.174:6825/3839005316] boot 2024-09-20T18:25:11.702 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: osdmap e56: 8 total, 8 up, 8 in 2024-09-20T18:25:11.702 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:25:11.702 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1250289087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-20T18:25:11.702 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:11 smithi174 ceph-mon[25682]: osdmap e57: 8 total, 8 up, 8 in 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.655 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:25:12.656 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:25:12.656 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:25:12.656 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.656 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:12 smithi012 ceph-mon[23745]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:25:12.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:25:12.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:25:12.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:25:12.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:12 smithi174 ceph-mon[25682]: osdmap e58: 8 total, 8 up, 8 in 2024-09-20T18:25:13.341 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:25:13.341 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":59,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","created":"2024-09-20T18:19:11.186571+0000","modified":"2024-09-20T18:25:12.861299+0000","last_up_change":"2024-09-20T18:25:10.364555+0000","last_in_change":"2024-09-20T18:24:52.994504+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T18:23:13.423037+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"123d122c-8976-4ecd-890a-396e77e8a687","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6803","nonce":459253037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6805","nonce":459253037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6809","nonce":459253037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6807","nonce":459253037}]},"public_addr":"172.21.15.12:6803/459253037","cluster_addr":"172.21.15.12:6805/459253037","heartbeat_back_addr":"172.21.15.12:6809/459253037","heartbeat_front_addr":"172.21.15.12:6807/459253037","state":["exists","up"]},{"osd":1,"uuid":"0d43fcc9-fb99-42f5-8e11-5f0ffe48d26c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6811","nonce":4097366944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6813","nonce":4097366944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6817","nonce":4097366944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6815","nonce":4097366944}]},"public_addr":"172.21.15.12:6811/4097366944","cluster_addr":"172.21.15.12:6813/4097366944","heartbeat_back_addr":"172.21.15.12:6817/4097366944","heartbeat_front_addr":"172.21.15.12:6815/4097366944","state":["exists","up"]},{"osd":2,"uuid":"36737f27-549f-4fce-98ba-c1e03dd4e84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6819","nonce":131798993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6821","nonce":131798993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6825","nonce":131798993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6823","nonce":131798993}]},"public_addr":"172.21.15.12:6819/131798993","cluster_addr":"172.21.15.12:6821/131798993","heartbeat_back_addr":"172.21.15.12:6825/131798993","heartbeat_front_addr":"172.21.15.12:6823/131798993","state":["exists","up"]},{"osd":3,"uuid":"dde6071d-ec62-461f-8ce4-d9d43c296320","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6827","nonce":3312213100}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6829","nonce":3312213100}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6833","nonce":3312213100}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6831","nonce":3312213100}]},"public_addr":"172.21.15.12:6827/3312213100","cluster_addr":"172.21.15.12:6829/3312213100","heartbeat_back_addr":"172.21.15.12:6833/3312213100","heartbeat_front_addr":"172.21.15.12:6831/3312213100","state":["exists","up"]},{"osd":4,"uuid":"41c03da2-744a-451b-95a2-ae934bba44cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6801","nonce":2768749008}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6803","nonce":2768749008}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6807","nonce":2768749008}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6805","nonce":2768749008}]},"public_addr":"172.21.15.174:6801/2768749008","cluster_addr":"172.21.15.174:6803/2768749008","heartbeat_back_addr":"172.21.15.174:6807/2768749008","heartbeat_front_addr":"172.21.15.174:6805/2768749008","state":["exists","up"]},{"osd":5,"uuid":"3375f2c1-371d-4256-b5fd-1bf87b2aeea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6809","nonce":3359326262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6811","nonce":3359326262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6815","nonce":3359326262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6813","nonce":3359326262}]},"public_addr":"172.21.15.174:6809/3359326262","cluster_addr":"172.21.15.174:6811/3359326262","heartbeat_back_addr":"172.21.15.174:6815/3359326262","heartbeat_front_addr":"172.21.15.174:6813/3359326262","state":["exists","up"]},{"osd":6,"uuid":"abf17c05-829d-4c3e-abb5-a1ffa2bdd583","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6817","nonce":1588901062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6819","nonce":1588901062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6823","nonce":1588901062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6821","nonce":1588901062}]},"public_addr":"172.21.15.174:6817/1588901062","cluster_addr":"172.21.15.174:6819/1588901062","heartbeat_back_addr":"172.21.15.174:6823/1588901062","heartbeat_front_addr":"172.21.15.174:6821/1588901062","state":["exists","up"]},{"osd":7,"uuid":"11ca6b60-4c5f-452d-ac4d-b6be024590a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6825","nonce":3839005316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6827","nonce":3839005316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6831","nonce":3839005316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6829","nonce":3839005316}]},"public_addr":"172.21.15.174:6825/3839005316","cluster_addr":"172.21.15.174:6827/3839005316","heartbeat_back_addr":"172.21.15.174:6831/3839005316","heartbeat_front_addr":"172.21.15.174:6829/3839005316","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:22:09.772675+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:22:41.770662+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:09.187265+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:35.827325+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:57.396435+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:24:20.761804+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:24:44.228481+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:25:08.475774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:6801/3010192060":"2024-09-21T18:20:58.851351+0000","172.21.15.12:6800/3010192060":"2024-09-21T18:20:58.851351+0000","172.21.15.12:6801/784058544":"2024-09-21T18:20:07.450384+0000","172.21.15.12:6800/784058544":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/470726279":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/1110941613":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/1370600009":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/1627970597":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/2953430076":"2024-09-21T18:19:37.124807+0000","172.21.15.12:0/372443117":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/3302964375":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/1097042779":"2024-09-21T18:19:37.124807+0000","172.21.15.12:6800/1053685683":"2024-09-21T18:19:37.124807+0000","172.21.15.12:6801/1053685683":"2024-09-21T18:19:37.124807+0000","172.21.15.12:0/403170997":"2024-09-21T18:19:37.124807+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T18:25:13.718 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:13 smithi012 ceph-mon[23745]: Detected new or changed devices on smithi174 2024-09-20T18:25:13.718 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:13 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi174 to 3984M 2024-09-20T18:25:13.718 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:13 smithi012 ceph-mon[23745]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T18:25:13.718 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:13 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1810901780' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T18:25:13.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:13 smithi174 ceph-mon[25682]: Detected new or changed devices on smithi174 2024-09-20T18:25:13.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:13 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi174 to 3984M 2024-09-20T18:25:13.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:13 smithi174 ceph-mon[25682]: osdmap e59: 8 total, 8 up, 8 in 2024-09-20T18:25:13.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:13 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1810901780' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T18:25:14.120 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-20T18:23:13.423037+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-20T18:25:14.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-20T18:25:14.335 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:14.699 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:14 smithi012 ceph-mon[23745]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-20T18:25:14.699 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:14 smithi012 ceph-mon[23745]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T18:25:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:14 smithi174 ceph-mon[25682]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-20T18:25:14.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:14 smithi174 ceph-mon[25682]: osdmap e60: 8 total, 8 up, 8 in 2024-09-20T18:25:16.268 INFO:teuthology.orchestra.run.smithi012.stdout:pg_num: 1 2024-09-20T18:25:16.650 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:16 smithi012 ceph-mon[23745]: pgmap v176: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s, 0 objects/s recovering 2024-09-20T18:25:16.650 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:16 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1003984239' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T18:25:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:16 smithi174 ceph-mon[25682]: pgmap v176: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s, 0 objects/s recovering 2024-09-20T18:25:16.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:16 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1003984239' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-20T18:25:17.098 INFO:tasks.cephadm:Setting up client nodes... 2024-09-20T18:25:17.098 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-20T18:25:18.643 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:18 smithi012 ceph-mon[23745]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T18:25:18.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:18 smithi174 ceph-mon[25682]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-20T18:25:19.307 INFO:teuthology.orchestra.run.smithi012.stdout:[client.0] 2024-09-20T18:25:19.307 INFO:teuthology.orchestra.run.smithi012.stdout: key = AQAPvu1mWixXERAAb9n7bp9mgeDjdbHmAFnJ1A== 2024-09-20T18:25:19.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:19 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3229432625' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T18:25:19.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:19 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3229432625' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T18:25:19.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:19 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3229432625' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-20T18:25:19.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:19 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3229432625' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-20T18:25:19.961 DEBUG:teuthology.orchestra.run.smithi012:> set -ex 2024-09-20T18:25:19.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-20T18:25:19.961 DEBUG:teuthology.orchestra.run.smithi012:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-20T18:25:20.004 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph config log 1 --format=json 2024-09-20T18:25:20.261 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:20.717 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:20 smithi012 ceph-mon[23745]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-20T18:25:20.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:20 smithi174 ceph-mon[25682]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-09-20T18:25:21.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:21 smithi012 ceph-mon[23745]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:22.089 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:25:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:21 smithi174 ceph-mon[25682]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:22.765 INFO:teuthology.orchestra.run.smithi012.stdout:[{"version":21,"timestamp":"2024-09-20T18:25:11.726803+0000","name":"","changes":[{"name":"osd/host:smithi174/osd_memory_target","previous_value":"5570054963","new_value":"4177541222"}]}] 2024-09-20T18:25:22.766 INFO:tasks.ceph_manager:config epoch is 21 2024-09-20T18:25:22.766 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-20T18:25:22.766 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-20T18:25:22.766 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph mgr dump --format=json 2024-09-20T18:25:22.969 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:22 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3965214099' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-20T18:25:22.988 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:23.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:22 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3965214099' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-20T18:25:24.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:23 smithi012 ceph-mon[23745]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:24.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:23 smithi174 ceph-mon[25682]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:24.869 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:25:25.191 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:24 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/535113617' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T18:25:25.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:24 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/535113617' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-20T18:25:25.663 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi012.xkxxrd","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6800","nonce":2571044339},{"type":"v1","addr":"172.21.15.12:6801","nonce":2571044339}]},"active_addr":"172.21.15.12:6801/2571044339","active_change":"2024-09-20T18:20:58.851728+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi174.nxkptu","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.12:8443/","prometheus":"http://172.21.15.12:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2739385043}]},{"addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":2036176528}]},{"addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":53367472}]},{"addrvec":[{"type":"v2","addr":"172.21.15.12:0","nonce":120077153}]}]}} 2024-09-20T18:25:25.666 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-20T18:25:25.666 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-20T18:25:25.666 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T18:25:25.897 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:25.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:25 smithi012 ceph-mon[23745]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:26.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:25 smithi174 ceph-mon[25682]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:27.653 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:25:27.654 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":60,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","created":"2024-09-20T18:19:11.186571+0000","modified":"2024-09-20T18:25:13.864149+0000","last_up_change":"2024-09-20T18:25:10.364555+0000","last_in_change":"2024-09-20T18:24:52.994504+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T18:23:13.423037+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"123d122c-8976-4ecd-890a-396e77e8a687","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6803","nonce":459253037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6805","nonce":459253037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6809","nonce":459253037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6807","nonce":459253037}]},"public_addr":"172.21.15.12:6803/459253037","cluster_addr":"172.21.15.12:6805/459253037","heartbeat_back_addr":"172.21.15.12:6809/459253037","heartbeat_front_addr":"172.21.15.12:6807/459253037","state":["exists","up"]},{"osd":1,"uuid":"0d43fcc9-fb99-42f5-8e11-5f0ffe48d26c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6811","nonce":4097366944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6813","nonce":4097366944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6817","nonce":4097366944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6815","nonce":4097366944}]},"public_addr":"172.21.15.12:6811/4097366944","cluster_addr":"172.21.15.12:6813/4097366944","heartbeat_back_addr":"172.21.15.12:6817/4097366944","heartbeat_front_addr":"172.21.15.12:6815/4097366944","state":["exists","up"]},{"osd":2,"uuid":"36737f27-549f-4fce-98ba-c1e03dd4e84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6819","nonce":131798993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6821","nonce":131798993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6825","nonce":131798993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6823","nonce":131798993}]},"public_addr":"172.21.15.12:6819/131798993","cluster_addr":"172.21.15.12:6821/131798993","heartbeat_back_addr":"172.21.15.12:6825/131798993","heartbeat_front_addr":"172.21.15.12:6823/131798993","state":["exists","up"]},{"osd":3,"uuid":"dde6071d-ec62-461f-8ce4-d9d43c296320","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6827","nonce":3312213100}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6829","nonce":3312213100}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6833","nonce":3312213100}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6831","nonce":3312213100}]},"public_addr":"172.21.15.12:6827/3312213100","cluster_addr":"172.21.15.12:6829/3312213100","heartbeat_back_addr":"172.21.15.12:6833/3312213100","heartbeat_front_addr":"172.21.15.12:6831/3312213100","state":["exists","up"]},{"osd":4,"uuid":"41c03da2-744a-451b-95a2-ae934bba44cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6801","nonce":2768749008}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6803","nonce":2768749008}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6807","nonce":2768749008}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6805","nonce":2768749008}]},"public_addr":"172.21.15.174:6801/2768749008","cluster_addr":"172.21.15.174:6803/2768749008","heartbeat_back_addr":"172.21.15.174:6807/2768749008","heartbeat_front_addr":"172.21.15.174:6805/2768749008","state":["exists","up"]},{"osd":5,"uuid":"3375f2c1-371d-4256-b5fd-1bf87b2aeea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6809","nonce":3359326262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6811","nonce":3359326262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6815","nonce":3359326262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6813","nonce":3359326262}]},"public_addr":"172.21.15.174:6809/3359326262","cluster_addr":"172.21.15.174:6811/3359326262","heartbeat_back_addr":"172.21.15.174:6815/3359326262","heartbeat_front_addr":"172.21.15.174:6813/3359326262","state":["exists","up"]},{"osd":6,"uuid":"abf17c05-829d-4c3e-abb5-a1ffa2bdd583","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6817","nonce":1588901062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6819","nonce":1588901062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6823","nonce":1588901062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6821","nonce":1588901062}]},"public_addr":"172.21.15.174:6817/1588901062","cluster_addr":"172.21.15.174:6819/1588901062","heartbeat_back_addr":"172.21.15.174:6823/1588901062","heartbeat_front_addr":"172.21.15.174:6821/1588901062","state":["exists","up"]},{"osd":7,"uuid":"11ca6b60-4c5f-452d-ac4d-b6be024590a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6825","nonce":3839005316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6827","nonce":3839005316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6831","nonce":3839005316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6829","nonce":3839005316}]},"public_addr":"172.21.15.174:6825/3839005316","cluster_addr":"172.21.15.174:6827/3839005316","heartbeat_back_addr":"172.21.15.174:6831/3839005316","heartbeat_front_addr":"172.21.15.174:6829/3839005316","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:22:09.772675+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:22:41.770662+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:09.187265+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:35.827325+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:57.396435+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:24:20.761804+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:24:44.228481+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:25:08.475774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:6801/3010192060":"2024-09-21T18:20:58.851351+0000","172.21.15.12:6800/3010192060":"2024-09-21T18:20:58.851351+0000","172.21.15.12:6801/784058544":"2024-09-21T18:20:07.450384+0000","172.21.15.12:6800/784058544":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/470726279":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/1110941613":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/1370600009":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/1627970597":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/2953430076":"2024-09-21T18:19:37.124807+0000","172.21.15.12:0/372443117":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/3302964375":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/1097042779":"2024-09-21T18:19:37.124807+0000","172.21.15.12:6800/1053685683":"2024-09-21T18:19:37.124807+0000","172.21.15.12:6801/1053685683":"2024-09-21T18:19:37.124807+0000","172.21.15.12:0/403170997":"2024-09-21T18:19:37.124807+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T18:25:28.405 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:28 smithi012 ceph-mon[23745]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:28.405 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:28 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2520211609' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T18:25:28.434 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-20T18:25:28.434 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd dump --format=json 2024-09-20T18:25:28.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:28 smithi174 ceph-mon[25682]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:28.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:28 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2520211609' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T18:25:28.652 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:30.118 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:30 smithi012 ceph-mon[23745]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:30.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:30 smithi174 ceph-mon[25682]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:30.613 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:25:30.614 INFO:teuthology.orchestra.run.smithi012.stdout:{"epoch":60,"fsid":"c01c9b48-777c-11ef-baf5-efdc52797490","created":"2024-09-20T18:19:11.186571+0000","modified":"2024-09-20T18:25:13.864149+0000","last_up_change":"2024-09-20T18:25:10.364555+0000","last_in_change":"2024-09-20T18:24:52.994504+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-20T18:23:13.423037+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"123d122c-8976-4ecd-890a-396e77e8a687","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6802","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6803","nonce":459253037}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6804","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6805","nonce":459253037}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6808","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6809","nonce":459253037}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6806","nonce":459253037},{"type":"v1","addr":"172.21.15.12:6807","nonce":459253037}]},"public_addr":"172.21.15.12:6803/459253037","cluster_addr":"172.21.15.12:6805/459253037","heartbeat_back_addr":"172.21.15.12:6809/459253037","heartbeat_front_addr":"172.21.15.12:6807/459253037","state":["exists","up"]},{"osd":1,"uuid":"0d43fcc9-fb99-42f5-8e11-5f0ffe48d26c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6810","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6811","nonce":4097366944}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6812","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6813","nonce":4097366944}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6816","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6817","nonce":4097366944}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6814","nonce":4097366944},{"type":"v1","addr":"172.21.15.12:6815","nonce":4097366944}]},"public_addr":"172.21.15.12:6811/4097366944","cluster_addr":"172.21.15.12:6813/4097366944","heartbeat_back_addr":"172.21.15.12:6817/4097366944","heartbeat_front_addr":"172.21.15.12:6815/4097366944","state":["exists","up"]},{"osd":2,"uuid":"36737f27-549f-4fce-98ba-c1e03dd4e84a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6818","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6819","nonce":131798993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6820","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6821","nonce":131798993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6824","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6825","nonce":131798993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6822","nonce":131798993},{"type":"v1","addr":"172.21.15.12:6823","nonce":131798993}]},"public_addr":"172.21.15.12:6819/131798993","cluster_addr":"172.21.15.12:6821/131798993","heartbeat_back_addr":"172.21.15.12:6825/131798993","heartbeat_front_addr":"172.21.15.12:6823/131798993","state":["exists","up"]},{"osd":3,"uuid":"dde6071d-ec62-461f-8ce4-d9d43c296320","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6826","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6827","nonce":3312213100}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6828","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6829","nonce":3312213100}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6832","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6833","nonce":3312213100}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.12:6830","nonce":3312213100},{"type":"v1","addr":"172.21.15.12:6831","nonce":3312213100}]},"public_addr":"172.21.15.12:6827/3312213100","cluster_addr":"172.21.15.12:6829/3312213100","heartbeat_back_addr":"172.21.15.12:6833/3312213100","heartbeat_front_addr":"172.21.15.12:6831/3312213100","state":["exists","up"]},{"osd":4,"uuid":"41c03da2-744a-451b-95a2-ae934bba44cc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6800","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6801","nonce":2768749008}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6802","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6803","nonce":2768749008}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6806","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6807","nonce":2768749008}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6804","nonce":2768749008},{"type":"v1","addr":"172.21.15.174:6805","nonce":2768749008}]},"public_addr":"172.21.15.174:6801/2768749008","cluster_addr":"172.21.15.174:6803/2768749008","heartbeat_back_addr":"172.21.15.174:6807/2768749008","heartbeat_front_addr":"172.21.15.174:6805/2768749008","state":["exists","up"]},{"osd":5,"uuid":"3375f2c1-371d-4256-b5fd-1bf87b2aeea2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6808","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6809","nonce":3359326262}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6810","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6811","nonce":3359326262}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6814","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6815","nonce":3359326262}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6812","nonce":3359326262},{"type":"v1","addr":"172.21.15.174:6813","nonce":3359326262}]},"public_addr":"172.21.15.174:6809/3359326262","cluster_addr":"172.21.15.174:6811/3359326262","heartbeat_back_addr":"172.21.15.174:6815/3359326262","heartbeat_front_addr":"172.21.15.174:6813/3359326262","state":["exists","up"]},{"osd":6,"uuid":"abf17c05-829d-4c3e-abb5-a1ffa2bdd583","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6816","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6817","nonce":1588901062}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6818","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6819","nonce":1588901062}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6822","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6823","nonce":1588901062}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6820","nonce":1588901062},{"type":"v1","addr":"172.21.15.174:6821","nonce":1588901062}]},"public_addr":"172.21.15.174:6817/1588901062","cluster_addr":"172.21.15.174:6819/1588901062","heartbeat_back_addr":"172.21.15.174:6823/1588901062","heartbeat_front_addr":"172.21.15.174:6821/1588901062","state":["exists","up"]},{"osd":7,"uuid":"11ca6b60-4c5f-452d-ac4d-b6be024590a6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6824","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6825","nonce":3839005316}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6826","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6827","nonce":3839005316}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6830","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6831","nonce":3839005316}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.174:6828","nonce":3839005316},{"type":"v1","addr":"172.21.15.174:6829","nonce":3839005316}]},"public_addr":"172.21.15.174:6825/3839005316","cluster_addr":"172.21.15.174:6827/3839005316","heartbeat_back_addr":"172.21.15.174:6831/3839005316","heartbeat_front_addr":"172.21.15.174:6829/3839005316","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:22:09.772675+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:22:41.770662+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:09.187265+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:35.827325+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:23:57.396435+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:24:20.761804+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:24:44.228481+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-20T18:25:08.475774+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.12:6801/3010192060":"2024-09-21T18:20:58.851351+0000","172.21.15.12:6800/3010192060":"2024-09-21T18:20:58.851351+0000","172.21.15.12:6801/784058544":"2024-09-21T18:20:07.450384+0000","172.21.15.12:6800/784058544":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/470726279":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/1110941613":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/1370600009":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/1627970597":"2024-09-21T18:20:07.450384+0000","172.21.15.12:0/2953430076":"2024-09-21T18:19:37.124807+0000","172.21.15.12:0/372443117":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/3302964375":"2024-09-21T18:20:58.851351+0000","172.21.15.12:0/1097042779":"2024-09-21T18:19:37.124807+0000","172.21.15.12:6800/1053685683":"2024-09-21T18:19:37.124807+0000","172.21.15.12:6801/1053685683":"2024-09-21T18:19:37.124807+0000","172.21.15.12:0/403170997":"2024-09-21T18:19:37.124807+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-20T18:25:31.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-20T18:25:31.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-20T18:25:31.178 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-20T18:25:31.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-20T18:25:31.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-20T18:25:31.179 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-20T18:25:31.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-20T18:25:31.180 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-20T18:25:31.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:31 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/659429572' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T18:25:31.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:31 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/659429572' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-20T18:25:31.598 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:31.598 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:31.598 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:31.602 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:31.602 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:31.602 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:31.604 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:31.615 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:32.309 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:32 smithi012 ceph-mon[23745]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:32.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:32 smithi174 ceph-mon[25682]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:34.161 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:34 smithi012 ceph-mon[23745]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:34.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:34 smithi174 ceph-mon[25682]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:36.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:36 smithi012 ceph-mon[23745]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:36.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:36 smithi174 ceph-mon[25682]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:38.262 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:38 smithi012 ceph-mon[23745]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:38.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:38 smithi174 ceph-mon[25682]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:40.309 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:40 smithi012 ceph-mon[23745]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:40.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:40 smithi174 ceph-mon[25682]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:42.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:41 smithi012 ceph-mon[23745]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:42.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:41 smithi174 ceph-mon[25682]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:43.354 INFO:teuthology.orchestra.run.smithi012.stdout:137438953493 2024-09-20T18:25:43.354 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-20T18:25:44.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:44 smithi012 ceph-mon[23745]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:44.245 INFO:teuthology.orchestra.run.smithi012.stdout:240518168585 2024-09-20T18:25:44.245 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-20T18:25:44.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:44 smithi174 ceph-mon[25682]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:44.834 INFO:teuthology.orchestra.run.smithi012.stdout:171798691859 2024-09-20T18:25:44.834 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-20T18:25:45.915 INFO:teuthology.orchestra.run.smithi012.stdout:60129542183 2024-09-20T18:25:45.915 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-20T18:25:46.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:46 smithi012 ceph-mon[23745]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:46 smithi174 ceph-mon[25682]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:47.449 INFO:teuthology.orchestra.run.smithi012.stdout:38654705709 2024-09-20T18:25:47.450 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-20T18:25:47.720 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:47.720 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:47.721 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:47.722 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:48.151 INFO:teuthology.orchestra.run.smithi012.stdout:206158430222 2024-09-20T18:25:48.152 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-20T18:25:48.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:48 smithi174 ceph-mon[25682]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:48.323 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:48 smithi012 ceph-mon[23745]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:48.513 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:48.927 INFO:teuthology.orchestra.run.smithi012.stdout:81604378658 2024-09-20T18:25:48.927 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-20T18:25:50.141 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:50.304 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:50 smithi012 ceph-mon[23745]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:50.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:50 smithi174 ceph-mon[25682]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:51.806 INFO:teuthology.orchestra.run.smithi012.stdout:115964117020 2024-09-20T18:25:51.807 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-20T18:25:51.919 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:51 smithi012 ceph-mon[23745]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:51 smithi174 ceph-mon[25682]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:53.943 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:54.035 INFO:teuthology.orchestra.run.smithi012.stdout:38654705711 2024-09-20T18:25:54.437 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:54 smithi012 ceph-mon[23745]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:54.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:54 smithi174 ceph-mon[25682]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:55.396 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:55 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/673135022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T18:25:55.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:55 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/673135022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-20T18:25:56.306 INFO:teuthology.orchestra.run.smithi012.stdout:137438953497 2024-09-20T18:25:56.307 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:56 smithi012 ceph-mon[23745]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:56.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:56 smithi174 ceph-mon[25682]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:57.287 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:57 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/525854419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T18:25:57.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:57 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/525854419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-20T18:25:57.592 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953493 got 137438953497 for osd.4 2024-09-20T18:25:57.592 DEBUG:teuthology.parallel:result is None 2024-09-20T18:25:58.061 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:25:58.291 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:58 smithi012 ceph-mon[23745]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:58.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:58 smithi174 ceph-mon[25682]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:25:58.699 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705711 for osd.0 2024-09-20T18:25:58.699 DEBUG:teuthology.parallel:result is None 2024-09-20T18:25:59.208 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:25:59.208 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:25:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:25:59.438 INFO:teuthology.orchestra.run.smithi012.stdout:206158430226 2024-09-20T18:25:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:25:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:25:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:26:00.114 INFO:teuthology.orchestra.run.smithi012.stdout:171798691862 2024-09-20T18:26:00.309 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:00 smithi012 ceph-mon[23745]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:00.309 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:00 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2410293093' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T18:26:00.314 INFO:teuthology.orchestra.run.smithi012.stdout:60129542186 2024-09-20T18:26:00.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:00 smithi174 ceph-mon[25682]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:00.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:00 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2410293093' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-20T18:26:00.928 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691859 got 171798691862 for osd.5 2024-09-20T18:26:00.929 DEBUG:teuthology.parallel:result is None 2024-09-20T18:26:01.155 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:01 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1565342314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T18:26:01.155 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:01 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3217928073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T18:26:01.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:01 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1565342314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-20T18:26:01.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:01 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3217928073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-20T18:26:02.331 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:02 smithi012 ceph-mon[23745]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:02.332 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:02 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1346848446' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T18:26:02.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:02 smithi174 ceph-mon[25682]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:02.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:02 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1346848446' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-20T18:26:02.631 INFO:teuthology.orchestra.run.smithi012.stdout:240518168589 2024-09-20T18:26:02.757 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430222 got 206158430226 for osd.6 2024-09-20T18:26:02.757 DEBUG:teuthology.parallel:result is None 2024-09-20T18:26:03.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:03 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/688123789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T18:26:03.363 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168589 for osd.7 2024-09-20T18:26:03.364 DEBUG:teuthology.parallel:result is None 2024-09-20T18:26:03.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:03 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/688123789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-20T18:26:03.874 INFO:teuthology.orchestra.run.smithi012.stdout:115964117023 2024-09-20T18:26:04.204 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:04 smithi012 ceph-mon[23745]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:04.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:04 smithi174 ceph-mon[25682]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:04.549 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542186 for osd.1 2024-09-20T18:26:04.550 DEBUG:teuthology.parallel:result is None 2024-09-20T18:26:05.176 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117020 got 115964117023 for osd.3 2024-09-20T18:26:05.176 DEBUG:teuthology.parallel:result is None 2024-09-20T18:26:05.556 INFO:teuthology.orchestra.run.smithi012.stdout:81604378662 2024-09-20T18:26:06.191 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:06 smithi012 ceph-mon[23745]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:06.191 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:06 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3440854126' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T18:26:06.219 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378662 for osd.2 2024-09-20T18:26:06.219 DEBUG:teuthology.parallel:result is None 2024-09-20T18:26:06.219 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-20T18:26:06.219 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T18:26:06.442 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:26:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:06 smithi174 ceph-mon[25682]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:06 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3440854126' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-20T18:26:08.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:08 smithi012 ceph-mon[23745]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:08.265 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:26:08.267 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-09-20T18:26:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:08 smithi174 ceph-mon[25682]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:08.928 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-20T18:26:06.959154+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49080,"kb_used_data":6392,"kb_used_omap":0,"kb_used_meta":42624,"kb_avail":749912136,"statfs":{"total":767960285184,"available":767910027264,"internally_reserved":0,"allocated":6545408,"data_stored":3572560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43646976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002007"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":223,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-20T18:25:13.885341+0000","last_change":"2024-09-20T18:25:13.885341+0000","last_active":"2024-09-20T18:25:13.885341+0000","last_peered":"2024-09-20T18:25:13.885341+0000","last_clean":"2024-09-20T18:25:13.885341+0000","last_became_active":"2024-09-20T18:25:13.884922+0000","last_became_peered":"2024-09-20T18:25:13.884922+0000","last_unstale":"2024-09-20T18:25:13.885341+0000","last_undegraded":"2024-09-20T18:25:13.885341+0000","last_fullsized":"2024-09-20T18:25:13.885341+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T18:23:13.473185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T18:23:13.473185+0000","last_clean_scrub_stamp":"2024-09-20T18:23:13.473185+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:26:15.406734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.51200000000000001}]},{"osd":1,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.38}]},{"osd":2,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.53800000000000003}]},{"osd":3,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.52500000000000002}]},{"osd":4,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.56899999999999995}]},{"osd":5,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.499}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48399999999999999}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.33700000000000002},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.373,"15min":0.35499999999999998},"min":{"1min":0.32900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57899999999999996,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Fri Sep 20 18:25:17 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.39800000000000002,"15min":0.38900000000000001},"min":{"1min":0.35999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54500000000000004,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.40500000000000003,"15min":0.40100000000000002},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.46800000000000003}]},{"osd":3,"last update":"Fri Sep 20 18:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.436},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.47599999999999998,"15min":0.48399999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.60199999999999998,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.441}]},{"osd":4,"last update":"Fri Sep 20 18:25:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.625},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64100000000000001}]},{"osd":5,"last update":"Fri Sep 20 18:25:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59699999999999998}]},{"osd":6,"last update":"Fri Sep 20 18:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.55800000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63300000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 18:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.375,"15min":0.36399999999999999},"min":{"1min":0.26500000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.61699999999999999,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.436},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.37},"min":{"1min":0.28299999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.44900000000000001}]},{"osd":2,"last update":"Fri Sep 20 18:25:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39100000000000001,"15min":0.38500000000000001},"min":{"1min":0.27700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.42499999999999999,"15min":0.41699999999999998},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.36299999999999999}]},{"osd":3,"last update":"Fri Sep 20 18:25:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.66000000000000003,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.36199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64600000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.42099999999999999}]},{"osd":4,"last update":"Fri Sep 20 18:25:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.55600000000000005}]},{"osd":5,"last update":"Fri Sep 20 18:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.56999999999999995}]},{"osd":6,"last update":"Fri Sep 20 18:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:13 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.36199999999999999,"15min":0.35199999999999998},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.38900000000000001,"15min":0.38100000000000001},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.61399999999999999,"5min":0.622,"15min":0.622},"last":0.377}]},{"osd":1,"last update":"Fri Sep 20 18:25:13 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.38100000000000001,"15min":0.376},"min":{"1min":0.26300000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55500000000000005,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.28100000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.34100000000000003}]},{"osd":3,"last update":"Fri Sep 20 18:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.63700000000000001,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.47099999999999997,"15min":0.46899999999999997},"min":{"1min":0.38100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.41299999999999998}]},{"osd":4,"last update":"Fri Sep 20 18:26:02 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.62,"15min":0.62},"min":{"1min":0.42699999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79200000000000004,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.46300000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.48199999999999998}]},{"osd":5,"last update":"Fri Sep 20 18:25:23 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.49399999999999999}]},{"osd":6,"last update":"Fri Sep 20 18:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.441},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.47099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53100000000000003}]}]},{"osd":3,"up_from":27,"seq":115964117024,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:41 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.41699999999999998,"15min":0.41499999999999998},"min":{"1min":0.32500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.309},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.497,"15min":0.498},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.35699999999999998}]},{"osd":1,"last update":"Fri Sep 20 18:25:41 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42499999999999999,"15min":0.42199999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.44500000000000001,"15min":0.44700000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.371}]},{"osd":2,"last update":"Fri Sep 20 18:25:41 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.48799999999999999,"15min":0.495},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.59799999999999998,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.34899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.59799999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.34799999999999998}]},{"osd":4,"last update":"Fri Sep 20 18:25:00 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.625}]},{"osd":5,"last update":"Fri Sep 20 18:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.57099999999999995}]},{"osd":6,"last update":"Fri Sep 20 18:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.58699999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60099999999999998}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.65400000000000003,"15min":0.58699999999999997},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":11.804,"5min":11.804,"15min":11.804},"last":0.503},{"interface":"front","average":{"1min":1.042,"5min":0.65800000000000003,"15min":0.59399999999999997},"min":{"1min":0.40799999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.788,"5min":11.788,"15min":11.788},"last":0.42999999999999999}]},{"osd":1,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55300000000000005,"15min":0.55400000000000005},"min":{"1min":0.40100000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.749,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.55100000000000005}]},{"osd":2,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54800000000000004,"15min":0.54700000000000004},"min":{"1min":0.41899999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.70699999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.621},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.76700000000000002,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.53200000000000003}]},{"osd":3,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.438,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.74399999999999999,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.0680000000000001,"5min":0.67000000000000004,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":11.765000000000001,"5min":11.765000000000001,"15min":11.765000000000001},"last":0.57999999999999996}]},{"osd":5,"last update":"Fri Sep 20 18:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.64600000000000002}]},{"osd":6,"last update":"Fri Sep 20 18:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.65400000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]}]},{"osd":5,"up_from":40,"seq":171798691863,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.503}]},{"osd":1,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.55500000000000005}]},{"osd":2,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.60099999999999998}]},{"osd":3,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.621}]},{"osd":4,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.442},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.54200000000000004}]},{"osd":6,"last update":"Fri Sep 20 18:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.52400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T18:26:08.930 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph pg dump --format=json 2024-09-20T18:26:09.152 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:26:09.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:09 smithi012 ceph-mon[23745]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:09.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:09 smithi174 ceph-mon[25682]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:10.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:10 smithi012 ceph-mon[23745]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:10 smithi174 ceph-mon[25682]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:10.897 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:26:10.900 INFO:teuthology.orchestra.run.smithi012.stderr:dumped all 2024-09-20T18:26:11.522 INFO:teuthology.orchestra.run.smithi012.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-20T18:26:08.959524+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":49080,"kb_used_data":6392,"kb_used_omap":0,"kb_used_meta":42624,"kb_avail":749912136,"statfs":{"total":767960285184,"available":767910027264,"internally_reserved":0,"allocated":6545408,"data_stored":3572560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43646976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002085"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":223,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-20T18:25:13.885341+0000","last_change":"2024-09-20T18:25:13.885341+0000","last_active":"2024-09-20T18:25:13.885341+0000","last_peered":"2024-09-20T18:25:13.885341+0000","last_clean":"2024-09-20T18:25:13.885341+0000","last_became_active":"2024-09-20T18:25:13.884922+0000","last_became_peered":"2024-09-20T18:25:13.884922+0000","last_unstale":"2024-09-20T18:25:13.885341+0000","last_undegraded":"2024-09-20T18:25:13.885341+0000","last_fullsized":"2024-09-20T18:25:13.885341+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-20T18:23:13.473185+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-20T18:23:13.473185+0000","last_clean_scrub_stamp":"2024-09-20T18:23:13.473185+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T03:26:15.406734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56899999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35699999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.498}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]}]},{"osd":6,"up_from":48,"seq":206158430228,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6188,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738964,"statfs":{"total":95995035648,"available":95988699136,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.55000000000000004}]},{"osd":1,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.50600000000000001}]},{"osd":2,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.0580000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.66200000000000003}]},{"osd":3,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.0900000000000001,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.56399999999999995}]},{"osd":4,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.70199999999999996}]},{"osd":5,"last update":"Fri Sep 20 18:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.67300000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71299999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6372,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.35799999999999998,"15min":0.33700000000000002},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.629,"5min":0.629,"15min":0.629},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.373,"15min":0.35499999999999998},"min":{"1min":0.32900000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.57899999999999996,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Fri Sep 20 18:25:17 2024","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.39800000000000002,"15min":0.38900000000000001},"min":{"1min":0.35999999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.54500000000000004,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.40500000000000003,"15min":0.40100000000000002},"min":{"1min":0.33200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.46800000000000003}]},{"osd":3,"last update":"Fri Sep 20 18:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.436},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.47599999999999998,"15min":0.48399999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.60199999999999998,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.441}]},{"osd":4,"last update":"Fri Sep 20 18:25:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.625},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.64100000000000001}]},{"osd":5,"last update":"Fri Sep 20 18:25:23 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59699999999999998}]},{"osd":6,"last update":"Fri Sep 20 18:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.55800000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63300000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6956,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738196,"statfs":{"total":95995035648,"available":95987912704,"internally_reserved":0,"allocated":1150976,"data_stored":777150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 20 18:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.375,"15min":0.36399999999999999},"min":{"1min":0.26500000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.61699999999999999,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.37},"min":{"1min":0.28299999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.36399999999999999}]},{"osd":2,"last update":"Fri Sep 20 18:25:20 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.39100000000000001,"15min":0.38500000000000001},"min":{"1min":0.27700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.42499999999999999,"15min":0.41699999999999998},"min":{"1min":0.33300000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.44700000000000001}]},{"osd":3,"last update":"Fri Sep 20 18:25:38 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.66000000000000003,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.36199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64600000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.42999999999999999}]},{"osd":4,"last update":"Fri Sep 20 18:26:03 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.63100000000000001,"15min":0.63600000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.79700000000000004,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.51300000000000001,"5min":0.438,"15min":0.438},"max":{"1min":0.84599999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.59099999999999997}]},{"osd":5,"last update":"Fri Sep 20 18:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.628}]},{"osd":6,"last update":"Fri Sep 20 18:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56000000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54600000000000004}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:13 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.36199999999999999,"15min":0.35199999999999998},"min":{"1min":0.28499999999999998,"5min":0.245,"15min":0.245},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.441},{"interface":"front","average":{"1min":0.435,"5min":0.38900000000000001,"15min":0.38100000000000001},"min":{"1min":0.29499999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.61399999999999999,"5min":0.622,"15min":0.622},"last":0.42899999999999999}]},{"osd":1,"last update":"Fri Sep 20 18:25:13 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.38100000000000001,"15min":0.376},"min":{"1min":0.26300000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.55500000000000005,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.41099999999999998,"15min":0.40200000000000002},"min":{"1min":0.28100000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.49299999999999999}]},{"osd":3,"last update":"Fri Sep 20 18:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.36299999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.63700000000000001,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.47099999999999997,"15min":0.46899999999999997},"min":{"1min":0.38100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.47099999999999997}]},{"osd":4,"last update":"Fri Sep 20 18:26:02 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.62,"15min":0.62},"min":{"1min":0.42699999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.79200000000000004,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.46300000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.78800000000000003}]},{"osd":5,"last update":"Fri Sep 20 18:25:23 2024","interfaces":[{"interface":"back","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.71099999999999997}]},{"osd":6,"last update":"Fri Sep 20 18:25:50 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.80300000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.747}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:41 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.41699999999999998,"15min":0.41499999999999998},"min":{"1min":0.32500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.497,"15min":0.498},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.35299999999999998}]},{"osd":1,"last update":"Fri Sep 20 18:25:41 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42499999999999999,"15min":0.42199999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.307},{"interface":"front","average":{"1min":0.433,"5min":0.44500000000000001,"15min":0.44700000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.28100000000000003}]},{"osd":2,"last update":"Fri Sep 20 18:25:41 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.48799999999999999,"15min":0.495},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.59799999999999998,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.34899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.59799999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.32500000000000001}]},{"osd":4,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.59999999999999998,"15min":0.59599999999999997},"min":{"1min":0.45700000000000002,"5min":0.436,"15min":0.436},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.379},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.58999999999999997,"15min":0.57899999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.47999999999999998}]},{"osd":5,"last update":"Fri Sep 20 18:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.41699999999999998}]},{"osd":6,"last update":"Fri Sep 20 18:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]}]},{"osd":4,"up_from":32,"seq":137438953500,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.65400000000000003,"15min":0.58699999999999997},"min":{"1min":0.34000000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":11.804,"5min":11.804,"15min":11.804},"last":0.503},{"interface":"front","average":{"1min":1.042,"5min":0.65800000000000003,"15min":0.59399999999999997},"min":{"1min":0.40799999999999997,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.788,"5min":11.788,"15min":11.788},"last":0.42999999999999999}]},{"osd":1,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.55300000000000005,"15min":0.55400000000000005},"min":{"1min":0.40100000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.749,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.55100000000000005}]},{"osd":2,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54800000000000004,"15min":0.54700000000000004},"min":{"1min":0.41899999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.70699999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.621},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.56799999999999995,"15min":0.56399999999999995},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.76700000000000002,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.53200000000000003}]},{"osd":3,"last update":"Fri Sep 20 18:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55700000000000005,"15min":0.55500000000000005},"min":{"1min":0.438,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.74399999999999999,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.0680000000000001,"5min":0.67000000000000004,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":11.765000000000001,"5min":11.765000000000001,"15min":11.765000000000001},"last":0.57999999999999996}]},{"osd":5,"last update":"Fri Sep 20 18:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.64600000000000002}]},{"osd":6,"last update":"Fri Sep 20 18:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.65400000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67800000000000005}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.61299999999999999}]},{"osd":1,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.65900000000000003}]},{"osd":2,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.68700000000000006}]},{"osd":3,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.53700000000000003}]},{"osd":4,"last update":"Fri Sep 20 18:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.55200000000000005}]},{"osd":6,"last update":"Fri Sep 20 18:25:49 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.74199999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.629}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-20T18:26:11.523 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-20T18:26:11.523 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-20T18:26:11.523 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-20T18:26:11.523 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph health --format=json 2024-09-20T18:26:11.743 INFO:teuthology.orchestra.run.smithi012.stderr:Inferring config /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/mon.smithi012/config 2024-09-20T18:26:12.067 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:11 smithi174 ceph-mon[25682]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:12.067 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:11 smithi174 ceph-mon[25682]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:12.067 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:12.067 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:12.067 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:11 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:12.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:11 smithi012 ceph-mon[23745]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:12.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:11 smithi012 ceph-mon[23745]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:12.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:12.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:12.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:11 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:13.137 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:13.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:13.604 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:26:13.605 INFO:teuthology.orchestra.run.smithi012.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-20T18:26:14.167 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-20T18:26:14.167 INFO:tasks.cephadm:Setup complete, yielding 2024-09-20T18:26:14.167 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T18:26:14.175 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:26:14.175 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch status' 2024-09-20T18:26:14.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:14 smithi012 ceph-mon[23745]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:14.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:14 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/409019671' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T18:26:14.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:14 smithi174 ceph-mon[25682]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:14.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:14 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/409019671' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-20T18:26:16.166 INFO:teuthology.orchestra.run.smithi012.stdout:Backend: cephadm 2024-09-20T18:26:16.166 INFO:teuthology.orchestra.run.smithi012.stdout:Available: Yes 2024-09-20T18:26:16.166 INFO:teuthology.orchestra.run.smithi012.stdout:Paused: No 2024-09-20T18:26:16.166 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:16 smithi012 ceph-mon[23745]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:16.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:16 smithi174 ceph-mon[25682]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:16.842 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ps' 2024-09-20T18:26:17.442 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:17 smithi012 ceph-mon[23745]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:17.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:17 smithi174 ceph-mon[25682]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:18.329 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:18 smithi012 ceph-mon[23745]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:18.419 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:18 smithi174 ceph-mon[25682]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:18.735 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:26:18.735 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (4m) 2m ago 5m 17.6M - ba2b418f427c f9d7e2615065 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (5m) 2m ago 5m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (5m) 69s ago 5m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (4m) 2m ago 5m 42.6M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (7m) 2m ago 7m 447M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (5m) 69s ago 5m 413M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (7m) 2m ago 7m 41.6M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (5m) 69s ago 5m 42.2M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (5m) 2m ago 5m 19.3M - 1dbe0e931976 721d4388b70c 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (4m) 69s ago 4m 19.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (4m) 2m ago 4m 35.3M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (3m) 2m ago 3m 34.6M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (3m) 2m ago 3m 31.3M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (2m) 2m ago 2m 10.0M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (2m) 69s ago 2m 32.8M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:26:18.736 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (2m) 69s ago 2m 33.7M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:26:18.737 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (99s) 69s ago 98s 32.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:26:18.737 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (74s) 69s ago 74s 11.8M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:26:18.737 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (4m) 2m ago 4m 52.5M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:26:19.387 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch ls' 2024-09-20T18:26:20.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:20 smithi012 ceph-mon[23745]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:20.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:20 smithi012 ceph-mon[23745]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:20.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:20 smithi174 ceph-mon[25682]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:20.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:20 smithi174 ceph-mon[25682]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:21.332 INFO:teuthology.orchestra.run.smithi012.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-20T18:26:21.332 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-20T18:26:21.333 INFO:teuthology.orchestra.run.smithi012.stdout:crash 2/2 2m ago 6m * 2024-09-20T18:26:21.333 INFO:teuthology.orchestra.run.smithi012.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-20T18:26:21.333 INFO:teuthology.orchestra.run.smithi012.stdout:mgr 2/2 2m ago 6m count:2 2024-09-20T18:26:21.333 INFO:teuthology.orchestra.run.smithi012.stdout:mon 2/2 2m ago 5m smithi012:172.21.15.12=smithi012;smithi174:172.21.15.174=smithi174;count:2 2024-09-20T18:26:21.333 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-20T18:26:21.333 INFO:teuthology.orchestra.run.smithi012.stdout:osd 8 2m ago - 2024-09-20T18:26:21.333 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-20T18:26:21.991 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch host ls' 2024-09-20T18:26:22.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:21 smithi012 ceph-mon[23745]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:22.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:21 smithi012 ceph-mon[23745]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:21 smithi174 ceph-mon[25682]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:21 smithi174 ceph-mon[25682]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:24.034 INFO:teuthology.orchestra.run.smithi012.stdout:HOST ADDR LABELS STATUS 2024-09-20T18:26:24.034 INFO:teuthology.orchestra.run.smithi012.stdout:smithi012 172.21.15.12 2024-09-20T18:26:24.034 INFO:teuthology.orchestra.run.smithi012.stdout:smithi174 172.21.15.174 2024-09-20T18:26:24.034 INFO:teuthology.orchestra.run.smithi012.stdout:2 hosts in cluster 2024-09-20T18:26:24.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:24 smithi012 ceph-mon[23745]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:24.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:24 smithi174 ceph-mon[25682]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:24.691 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph orch device ls' 2024-09-20T18:26:25.325 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:25 smithi012 ceph-mon[23745]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:25.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:25 smithi174 ceph-mon[25682]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:26.287 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:26 smithi012 ceph-mon[23745]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:26.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:26 smithi174 ceph-mon[25682]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:26.734 INFO:teuthology.orchestra.run.smithi012.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-20T18:26:26.734 INFO:teuthology.orchestra.run.smithi012.stdout:smithi012 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5331005J400BGN 400G LVM detected, locked 2024-09-20T18:26:26.734 INFO:teuthology.orchestra.run.smithi012.stdout:smithi174 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT62050074400BGN 400G LVM detected, locked 2024-09-20T18:26:27.405 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T18:26:27.415 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:26:27.416 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-20T18:26:27.444 INFO:teuthology.orchestra.run.smithi012.stderr:+ systemctl stop nfs-server 2024-09-20T18:26:27.452 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi174.front.sepia.ceph.com 2024-09-20T18:26:27.453 DEBUG:teuthology.orchestra.run.smithi174:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-20T18:26:27.482 INFO:teuthology.orchestra.run.smithi174.stderr:+ systemctl stop nfs-server 2024-09-20T18:26:27.490 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T18:26:27.510 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:26:27.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph fs volume create foofs' 2024-09-20T18:26:28.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:28 smithi012 ceph-mon[23745]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:28.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:28 smithi012 ceph-mon[23745]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:28.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:28 smithi174 ceph-mon[25682]: from='client.14476 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:28.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:28 smithi174 ceph-mon[25682]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:30.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:30 smithi012 ceph-mon[23745]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:30.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:30 smithi012 ceph-mon[23745]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:30.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:30 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-20T18:26:30.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:30 smithi174 ceph-mon[25682]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:30.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:30 smithi174 ceph-mon[25682]: from='client.14480 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:30.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:30 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-20T18:26:31.444 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-20T18:26:31.444 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:31 smithi012 ceph-mon[23745]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T18:26:31.444 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-20T18:26:31.444 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:31 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[23741]: 2024-09-20T18:26:31.039+0000 7f73ff991700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:26:31.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-20T18:26:31.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:31 smithi174 ceph-mon[25682]: osdmap e61: 8 total, 8 up, 8 in 2024-09-20T18:26:31.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-20T18:26:31.740 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-20T18:26:31.755 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-20T18:26:31.756 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: pgmap v215: 33 pgs: 1 creating+peering, 1 active+clean, 31 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: fsmap foofs:0 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: Saving service mds.foofs spec with placement count:2 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:32.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:32.255 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:32 smithi012 ceph-mon[23745]: Deploying daemon mds.foofs.smithi012.jjsnmc on smithi012 2024-09-20T18:26:32.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: pgmap v215: 33 pgs: 1 creating+peering, 1 active+clean, 31 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: osdmap e62: 8 total, 8 up, 8 in 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: osdmap e63: 8 total, 8 up, 8 in 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: fsmap foofs:0 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: Saving service mds.foofs spec with placement count:2 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T18:26:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T18:26:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:32 smithi174 ceph-mon[25682]: Deploying daemon mds.foofs.smithi012.jjsnmc on smithi012 2024-09-20T18:26:33.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:33 smithi012 ceph-mon[23745]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T18:26:33.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:33 smithi174 ceph-mon[25682]: osdmap e64: 8 total, 8 up, 8 in 2024-09-20T18:26:34.353 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:34 smithi012 ceph-mon[23745]: pgmap v219: 65 pgs: 31 active+clean, 12 creating+peering, 22 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:34.353 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:34 smithi012 ceph-mon[23745]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T18:26:34.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:34 smithi174 ceph-mon[25682]: pgmap v219: 65 pgs: 31 active+clean, 12 creating+peering, 22 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:34.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:34 smithi174 ceph-mon[25682]: osdmap e65: 8 total, 8 up, 8 in 2024-09-20T18:26:35.127 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:26:35.128 INFO:teuthology.orchestra.run.smithi012.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T18:19:56.492926Z", "last_refresh": "2024-09-20T18:23:38.439041Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T18:19:49.875361Z", "last_refresh": "2024-09-20T18:23:38.439109Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T18:19:52.595112Z", "last_refresh": "2024-09-20T18:23:38.439174Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T18:26:31.062063Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T18:26:31.056849Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T18:19:48.070520Z", "last_refresh": "2024-09-20T18:23:38.438961Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi012:172.21.15.12=smithi012", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T18:20:41.415183Z", "last_refresh": "2024-09-20T18:23:38.438811Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T18:19:54.379599Z", "last_refresh": "2024-09-20T18:23:38.439238Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T18:23:38.439364Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T18:19:51.374551Z", "last_refresh": "2024-09-20T18:23:38.439301Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T18:26:36.092 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-20T18:26:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: pgmap v221: 65 pgs: 49 active+clean, 12 creating+peering, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T18:26:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T18:26:36.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: daemon mds.foofs.smithi012.jjsnmc assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: Cluster is now healthy 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: mds.? [v2:172.21.15.12:6834/1898222382,v1:172.21.15.12:6835/1898222382] up:boot 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:creating} 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"}]: dispatch 2024-09-20T18:26:36.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:36 smithi012 ceph-mon[23745]: daemon mds.foofs.smithi012.jjsnmc is now active in filesystem foofs as rank 0 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: pgmap v221: 65 pgs: 49 active+clean, 12 creating+peering, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: from='client.14484 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: daemon mds.foofs.smithi012.jjsnmc assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: Cluster is now healthy 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: mds.? [v2:172.21.15.12:6834/1898222382,v1:172.21.15.12:6835/1898222382] up:boot 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:creating} 2024-09-20T18:26:36.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"}]: dispatch 2024-09-20T18:26:36.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:36 smithi174 ceph-mon[25682]: daemon mds.foofs.smithi012.jjsnmc is now active in filesystem foofs as rank 0 2024-09-20T18:26:37.093 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T18:26:37.705 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:37 smithi012 ceph-mon[23745]: Deploying daemon mds.foofs.smithi174.zwzgjx on smithi174 2024-09-20T18:26:37.706 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:37 smithi012 ceph-mon[23745]: mds.? [v2:172.21.15.12:6834/1898222382,v1:172.21.15.12:6835/1898222382] up:active 2024-09-20T18:26:37.706 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:37 smithi012 ceph-mon[23745]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 2024-09-20T18:26:37.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:37 smithi174 ceph-mon[25682]: Deploying daemon mds.foofs.smithi174.zwzgjx on smithi174 2024-09-20T18:26:37.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:37 smithi174 ceph-mon[25682]: mds.? [v2:172.21.15.12:6834/1898222382,v1:172.21.15.12:6835/1898222382] up:active 2024-09-20T18:26:37.768 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:37 smithi174 ceph-mon[25682]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 2024-09-20T18:26:38.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:38 smithi174 ceph-mon[25682]: pgmap v222: 65 pgs: 60 active+clean, 5 creating+peering; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-09-20T18:26:38.815 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:38.816 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:38.816 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:38.816 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:38 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:38.855 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:38 smithi012 ceph-mon[23745]: pgmap v222: 65 pgs: 60 active+clean, 5 creating+peering; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-09-20T18:26:38.855 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:38.855 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:38.856 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:38.856 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:38 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:39.287 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:26:39.287 INFO:teuthology.orchestra.run.smithi012.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T18:19:56.492926Z", "last_refresh": "2024-09-20T18:23:38.439041Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T18:19:49.875361Z", "last_refresh": "2024-09-20T18:23:38.439109Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T18:19:52.595112Z", "last_refresh": "2024-09-20T18:23:38.439174Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T18:26:31.062063Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T18:26:31.056849Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T18:19:48.070520Z", "last_refresh": "2024-09-20T18:23:38.438961Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi012:172.21.15.12=smithi012", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T18:20:41.415183Z", "last_refresh": "2024-09-20T18:23:38.438811Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T18:19:54.379599Z", "last_refresh": "2024-09-20T18:23:38.439238Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T18:23:38.439364Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T18:19:51.374551Z", "last_refresh": "2024-09-20T18:23:38.439301Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T18:26:39.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:39 smithi012 ceph-mon[23745]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:boot 2024-09-20T18:26:39.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:39 smithi012 ceph-mon[23745]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:26:39.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:39 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"}]: dispatch 2024-09-20T18:26:39.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:39 smithi012 ceph-mon[23745]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:26:39.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:39 smithi012 ceph-mon[23745]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T18:26:39.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:39 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:39.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:39 smithi012 ceph-mon[23745]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:40.099 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-20T18:26:40.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:39 smithi174 ceph-mon[25682]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:boot 2024-09-20T18:26:40.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:39 smithi174 ceph-mon[25682]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:26:40.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:39 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"}]: dispatch 2024-09-20T18:26:40.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:39 smithi174 ceph-mon[25682]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:26:40.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:39 smithi174 ceph-mon[25682]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T18:26:40.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:39 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:40.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:39 smithi174 ceph-mon[25682]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:41.099 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T18:26:41.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:41 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:41.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:41 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:42.450 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:42 smithi012 ceph-mon[23745]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-20T18:26:42.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:42 smithi174 ceph-mon[25682]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-09-20T18:26:44.277 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:44 smithi012 ceph-mon[23745]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T18:26:44.278 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:44 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:44.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:44 smithi174 ceph-mon[25682]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-20T18:26:44.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:44 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:44.578 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:26:44.578 INFO:teuthology.orchestra.run.smithi012.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T18:19:56.492926Z", "last_refresh": "2024-09-20T18:23:38.439041Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T18:19:49.875361Z", "last_refresh": "2024-09-20T18:23:38.439109Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T18:19:52.595112Z", "last_refresh": "2024-09-20T18:23:38.439174Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T18:26:31.062063Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T18:26:31.056849Z", "last_refresh": "2024-09-20T18:26:40.086803Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T18:19:48.070520Z", "last_refresh": "2024-09-20T18:23:38.438961Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi012:172.21.15.12=smithi012", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T18:20:41.415183Z", "last_refresh": "2024-09-20T18:23:38.438811Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T18:19:54.379599Z", "last_refresh": "2024-09-20T18:23:38.439238Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T18:23:38.439364Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T18:19:51.374551Z", "last_refresh": "2024-09-20T18:23:38.439301Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T18:26:45.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:45 smithi012 ceph-mon[23745]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:45 smithi174 ceph-mon[25682]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:45.584 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-20T18:26:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:46 smithi012 ceph-mon[23745]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T18:26:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:46.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:46.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:46.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:46.474 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:46 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:46 smithi174 ceph-mon[25682]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T18:26:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:46 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:46.585 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T18:26:48.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:48 smithi174 ceph-mon[25682]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T18:26:48.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:48 smithi012 ceph-mon[23745]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T18:26:48.557 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:26:48.557 INFO:teuthology.orchestra.run.smithi012.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T18:19:56.492926Z", "last_refresh": "2024-09-20T18:26:45.802607Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T18:19:49.875361Z", "last_refresh": "2024-09-20T18:26:40.085838Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-20T18:19:52.595112Z", "last_refresh": "2024-09-20T18:26:45.802734Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-20T18:26:31.062063Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T18:26:31.056849Z", "last_refresh": "2024-09-20T18:26:40.086803Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T18:19:48.070520Z", "last_refresh": "2024-09-20T18:26:40.086060Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi012:172.21.15.12=smithi012", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T18:20:41.415183Z", "last_refresh": "2024-09-20T18:26:40.086220Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T18:19:54.379599Z", "last_refresh": "2024-09-20T18:26:40.086362Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-20T18:26:40.086444Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T18:19:51.374551Z", "last_refresh": "2024-09-20T18:26:45.802859Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T18:26:49.185 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-20T18:26:49.185 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T18:26:49.194 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:26:49.194 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-20T18:26:49.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:49 smithi012 ceph-mon[23745]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:49.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:49 smithi174 ceph-mon[25682]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:26:50.473 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:50 smithi012 ceph-mon[23745]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T18:26:50.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:50 smithi174 ceph-mon[25682]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-20T18:26:52.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:51 smithi012 ceph-mon[23745]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-20T18:26:52.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:51 smithi012 ceph-mon[23745]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:52.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:51 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-20T18:26:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:51 smithi174 ceph-mon[25682]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-20T18:26:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:51 smithi174 ceph-mon[25682]: from='client.14506 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:51 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-20T18:26:52.934 INFO:teuthology.orchestra.run.smithi012.stdout:NFS Cluster Created Successfully 2024-09-20T18:26:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:52 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-20T18:26:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:52 smithi012 ceph-mon[23745]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T18:26:53.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:52 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-20T18:26:53.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:52 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-20T18:26:53.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:52 smithi174 ceph-mon[25682]: osdmap e66: 8 total, 8 up, 8 in 2024-09-20T18:26:53.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:52 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-20T18:26:53.643 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-20T18:26:54.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-20T18:26:54.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T18:26:54.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: Saving service nfs.foo spec with placement count:2 2024-09-20T18:26:54.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:54.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: pgmap v232: 97 pgs: 11 creating+peering, 21 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: Creating key for client.nfs.foo.0.0.smithi012.nochla 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:26:54.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: Rados config object exists: conf-nfs.foo 2024-09-20T18:26:54.183 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: Creating key for client.nfs.foo.0.0.smithi012.nochla-rgw 2024-09-20T18:26:54.183 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T18:26:54.183 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T18:26:54.183 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:54.183 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:53 smithi012 ceph-mon[23745]: Deploying daemon nfs.foo.0.0.smithi012.nochla on smithi012 2024-09-20T18:26:54.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: osdmap e67: 8 total, 8 up, 8 in 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: Saving service nfs.foo spec with placement count:2 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: pgmap v232: 97 pgs: 11 creating+peering, 21 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: Creating key for client.nfs.foo.0.0.smithi012.nochla 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T18:26:54.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: Rados config object exists: conf-nfs.foo 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: Creating key for client.nfs.foo.0.0.smithi012.nochla-rgw 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:53 smithi174 ceph-mon[25682]: Deploying daemon nfs.foo.0.0.smithi012.nochla on smithi012 2024-09-20T18:26:55.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:54 smithi012 ceph-mon[23745]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T18:26:55.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:54 smithi012 ceph-mon[23745]: mgrmap e20: smithi012.xkxxrd(active, since 5m), standbys: smithi174.nxkptu 2024-09-20T18:26:55.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:54 smithi174 ceph-mon[25682]: osdmap e68: 8 total, 8 up, 8 in 2024-09-20T18:26:55.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:54 smithi174 ceph-mon[25682]: mgrmap e20: smithi012.xkxxrd(active, since 5m), standbys: smithi174.nxkptu 2024-09-20T18:26:56.120 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:55 smithi012 ceph-mon[23745]: pgmap v234: 97 pgs: 4 creating+activating, 11 creating+peering, 9 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T18:26:56.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:55 smithi174 ceph-mon[25682]: pgmap v234: 97 pgs: 4 creating+activating, 11 creating+peering, 9 unknown, 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T18:26:57.419 INFO:teuthology.orchestra.run.smithi012.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-20T18:26:57.419 INFO:teuthology.orchestra.run.smithi012.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-20T18:26:57.420 INFO:teuthology.orchestra.run.smithi012.stderr:Error EINVAL: invalid command 2024-09-20T18:26:57.768 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:26:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "bind": "/fake", 2024-09-20T18:26:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "fs": "foofs", 2024-09-20T18:26:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "path": "/", 2024-09-20T18:26:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "cluster": "foo", 2024-09-20T18:26:57.768 INFO:teuthology.orchestra.run.smithi012.stdout: "mode": "RW" 2024-09-20T18:26:57.768 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: pgmap v235: 97 pgs: 4 creating+activating, 11 creating+peering, 82 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T18:26:58.148 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-20T18:26:58.149 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:57 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-20T18:26:58.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:26:58.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj 2024-09-20T18:26:58.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: pgmap v235: 97 pgs: 4 creating+activating, 11 creating+peering, 82 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-20T18:26:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:57 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-20T18:26:58.493 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-20T18:26:59.143 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:58 smithi012 ceph-mon[23745]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:26:59.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:58 smithi174 ceph-mon[25682]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T18:27:00.056 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:26:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: pgmap v236: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-09-20T18:27:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:27:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:27:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-20T18:27:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:27:00.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-20T18:27:00.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-20T18:27:00.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:26:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:01.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:00 smithi012 ceph-mon[23745]: Rados config object exists: conf-nfs.foo 2024-09-20T18:27:01.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:00 smithi012 ceph-mon[23745]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj-rgw 2024-09-20T18:27:01.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:00 smithi012 ceph-mon[23745]: Deploying daemon nfs.foo.1.0.smithi174.yhunhj on smithi174 2024-09-20T18:27:01.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:00 smithi012 ceph-mon[23745]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:01.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:00 smithi174 ceph-mon[25682]: Rados config object exists: conf-nfs.foo 2024-09-20T18:27:01.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:00 smithi174 ceph-mon[25682]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj-rgw 2024-09-20T18:27:01.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:00 smithi174 ceph-mon[25682]: Deploying daemon nfs.foo.1.0.smithi174.yhunhj on smithi174 2024-09-20T18:27:01.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:00 smithi174 ceph-mon[25682]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:01.977 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:01 smithi174 ceph-mon[25682]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-09-20T18:27:02.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:01 smithi012 ceph-mon[23745]: pgmap v237: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-09-20T18:27:03.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:03 smithi012 ceph-mon[23745]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:03.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:03.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:27:03.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:03.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:03 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:27:03.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:03 smithi174 ceph-mon[25682]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:03.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:03.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:27:03.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:03.403 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:03 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:27:04.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:04 smithi012 ceph-mon[23745]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.9 KiB/s wr, 3 op/s 2024-09-20T18:27:04.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:04 smithi012 ceph-mon[23745]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:04.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:04 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:04.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:04 smithi174 ceph-mon[25682]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.9 KiB/s wr, 3 op/s 2024-09-20T18:27:04.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:04 smithi174 ceph-mon[25682]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:04.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:04 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:05.792 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:05 smithi012 ceph-mon[23745]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:05.792 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:05 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:06.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:05 smithi174 ceph-mon[25682]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:06.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:05 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:06.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:06 smithi012 ceph-mon[23745]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.4 KiB/s wr, 6 op/s 2024-09-20T18:27:07.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:06 smithi174 ceph-mon[25682]: pgmap v239: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 2.4 KiB/s wr, 6 op/s 2024-09-20T18:27:07.062 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-20T18:27:07.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:07.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:07.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:07.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:07.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:27:07.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:07.723 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:27:07.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:07.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:07 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:07.793 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T18:27:07.808 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:27:07.809 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-20T18:27:07.838 INFO:teuthology.orchestra.run.smithi012.stderr:+ mkdir /mnt/foo 2024-09-20T18:27:07.850 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-20T18:27:07.917 INFO:teuthology.orchestra.run.smithi012.stderr:++ hostname 2024-09-20T18:27:07.918 INFO:teuthology.orchestra.run.smithi012.stderr:+ mount -t nfs smithi012:/fake /mnt/foo -o sync 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:08.019 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:07 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:08.380 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-20T18:27:08.445 INFO:teuthology.orchestra.run.smithi012.stderr:+ echo test 2024-09-20T18:27:08.471 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-20T18:27:08.536 INFO:teuthology.orchestra.run.smithi012.stderr:+ sync 2024-09-20T18:27:08.781 INFO:teuthology.run_tasks:Running task parallel... 2024-09-20T18:27:08.797 INFO:teuthology.task.parallel:starting parallel... 2024-09-20T18:27:08.797 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-20T18:27:08.797 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-20T18:27:08.798 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:27:08.798 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-20T18:27:08.798 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-20T18:27:08.798 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-20T18:27:08.799 INFO:teuthology.task.exec:Executing custom commands... 2024-09-20T18:27:08.799 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:27:08.799 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-20T18:27:08.845 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:08 smithi012 ceph-mon[23745]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-20T18:27:08.846 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:08 smithi012 ceph-mon[23745]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:08.910 INFO:teuthology.orchestra.run.smithi012.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-20T18:27:08.910 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:27:08.910 INFO:teuthology.orchestra.run.smithi012.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-20T18:27:08.911 INFO:teuthology.orchestra.run.smithi012.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-20T18:27:09.011 INFO:teuthology.orchestra.run.smithi012.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-20T18:27:09.012 INFO:teuthology.orchestra.run.smithi012.stdout:releasing clients 2024-09-20T18:27:09.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:08 smithi174 ceph-mon[25682]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-20T18:27:09.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:08 smithi174 ceph-mon[25682]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:10.011 INFO:teuthology.orchestra.run.smithi012.stdout: 5 136 29.53 MB/sec warmup 1 sec latency 34.027 ms 2024-09-20T18:27:10.782 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:10 smithi012 ceph-mon[23745]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-20T18:27:11.011 INFO:teuthology.orchestra.run.smithi012.stdout: 5 297 27.48 MB/sec warmup 2 sec latency 23.572 ms 2024-09-20T18:27:11.018 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:10 smithi174 ceph-mon[25682]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 6 op/s 2024-09-20T18:27:12.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 421 24.69 MB/sec warmup 3 sec latency 54.919 ms 2024-09-20T18:27:12.067 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:11 smithi012 ceph-mon[23745]: pgmap v242: 97 pgs: 97 active+clean; 6.8 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 537 KiB/s wr, 13 op/s 2024-09-20T18:27:12.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:11 smithi174 ceph-mon[25682]: pgmap v242: 97 pgs: 97 active+clean; 6.8 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 537 KiB/s wr, 13 op/s 2024-09-20T18:27:12.391 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-20T18:27:13.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 537 23.26 MB/sec warmup 4 sec latency 54.342 ms 2024-09-20T18:27:14.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 789 21.43 MB/sec warmup 5 sec latency 23.748 ms 2024-09-20T18:27:14.120 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:14 smithi012 ceph-mon[23745]: pgmap v243: 97 pgs: 97 active+clean; 52 MiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.4 MiB/s wr, 208 op/s 2024-09-20T18:27:14.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:14 smithi174 ceph-mon[25682]: pgmap v243: 97 pgs: 97 active+clean; 52 MiB data, 99 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.4 MiB/s wr, 208 op/s 2024-09-20T18:27:15.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 1245 19.70 MB/sec warmup 6 sec latency 24.099 ms 2024-09-20T18:27:16.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 2134 19.11 MB/sec warmup 7 sec latency 22.653 ms 2024-09-20T18:27:16.137 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:16 smithi012 ceph-mon[23745]: pgmap v244: 97 pgs: 97 active+clean; 80 MiB data, 128 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.9 MiB/s wr, 297 op/s 2024-09-20T18:27:16.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:16 smithi174 ceph-mon[25682]: pgmap v244: 97 pgs: 97 active+clean; 80 MiB data, 128 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.9 MiB/s wr, 297 op/s 2024-09-20T18:27:17.002 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-20T18:27:17.011 INFO:teuthology.orchestra.run.smithi012.stdout: 5 2747 19.40 MB/sec warmup 8 sec latency 27.243 ms 2024-09-20T18:27:18.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 3335 19.84 MB/sec warmup 9 sec latency 28.909 ms 2024-09-20T18:27:18.187 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:18 smithi012 ceph-mon[23745]: pgmap v245: 97 pgs: 97 active+clean; 98 MiB data, 156 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 8.7 MiB/s wr, 399 op/s 2024-09-20T18:27:18.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:18 smithi174 ceph-mon[25682]: pgmap v245: 97 pgs: 97 active+clean; 98 MiB data, 156 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 8.7 MiB/s wr, 399 op/s 2024-09-20T18:27:19.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 3772 19.73 MB/sec warmup 10 sec latency 45.182 ms 2024-09-20T18:27:20.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 4007 18.20 MB/sec warmup 11 sec latency 41.144 ms 2024-09-20T18:27:20.248 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:20 smithi012 ceph-mon[23745]: pgmap v246: 97 pgs: 97 active+clean; 119 MiB data, 179 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 11 MiB/s wr, 564 op/s 2024-09-20T18:27:20.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:20 smithi174 ceph-mon[25682]: pgmap v246: 97 pgs: 97 active+clean; 119 MiB data, 179 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 11 MiB/s wr, 564 op/s 2024-09-20T18:27:21.013 INFO:teuthology.orchestra.run.smithi012.stdout: 5 4637 17.40 MB/sec warmup 12 sec latency 39.721 ms 2024-09-20T18:27:21.579 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-20T18:27:22.011 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:21 smithi012 ceph-mon[23745]: pgmap v247: 97 pgs: 97 active+clean; 122 MiB data, 200 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 12 MiB/s wr, 590 op/s 2024-09-20T18:27:22.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 5325 17.46 MB/sec warmup 13 sec latency 34.254 ms 2024-09-20T18:27:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:21 smithi174 ceph-mon[25682]: pgmap v247: 97 pgs: 97 active+clean; 122 MiB data, 200 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 12 MiB/s wr, 590 op/s 2024-09-20T18:27:23.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 5954 16.81 MB/sec warmup 14 sec latency 54.518 ms 2024-09-20T18:27:24.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 6450 17.17 MB/sec warmup 15 sec latency 33.528 ms 2024-09-20T18:27:24.293 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:24 smithi012 ceph-mon[23745]: pgmap v248: 97 pgs: 97 active+clean; 132 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 13 MiB/s wr, 829 op/s 2024-09-20T18:27:24.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:24 smithi174 ceph-mon[25682]: pgmap v248: 97 pgs: 97 active+clean; 132 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 13 MiB/s wr, 829 op/s 2024-09-20T18:27:25.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 6885 17.09 MB/sec warmup 16 sec latency 65.615 ms 2024-09-20T18:27:25.065 INFO:teuthology.orchestra.run.smithi012.stderr:module 'nfs' is already enabled 2024-09-20T18:27:25.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:25 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2071297075' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-20T18:27:25.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:25 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2071297075' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-20T18:27:26.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 7251 17.17 MB/sec warmup 17 sec latency 50.672 ms 2024-09-20T18:27:26.108 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:26 smithi012 ceph-mon[23745]: pgmap v249: 97 pgs: 97 active+clean; 132 MiB data, 269 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 10 MiB/s wr, 730 op/s 2024-09-20T18:27:26.108 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:26 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2071297075' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-20T18:27:26.108 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:26 smithi012 ceph-mon[23745]: mgrmap e21: smithi012.xkxxrd(active, since 6m), standbys: smithi174.nxkptu 2024-09-20T18:27:26.157 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-20T18:27:26.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:26 smithi174 ceph-mon[25682]: pgmap v249: 97 pgs: 97 active+clean; 132 MiB data, 269 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 10 MiB/s wr, 730 op/s 2024-09-20T18:27:26.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:26 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2071297075' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-20T18:27:26.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:26 smithi174 ceph-mon[25682]: mgrmap e21: smithi012.xkxxrd(active, since 6m), standbys: smithi174.nxkptu 2024-09-20T18:27:27.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 7434 16.41 MB/sec warmup 18 sec latency 49.794 ms 2024-09-20T18:27:28.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 7756 15.64 MB/sec warmup 19 sec latency 87.709 ms 2024-09-20T18:27:28.156 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:28 smithi012 ceph-mon[23745]: pgmap v250: 97 pgs: 97 active+clean; 132 MiB data, 296 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 9.0 MiB/s wr, 741 op/s 2024-09-20T18:27:28.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:28 smithi174 ceph-mon[25682]: pgmap v250: 97 pgs: 97 active+clean; 132 MiB data, 296 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 9.0 MiB/s wr, 741 op/s 2024-09-20T18:27:28.892 INFO:teuthology.orchestra.run.smithi012.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 2024-09-20T18:27:29.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 8089 15.15 MB/sec warmup 20 sec latency 67.336 ms 2024-09-20T18:27:29.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:29.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 2024-09-20T18:27:29.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:29.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:27:29.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:29.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:27:29.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: pgmap v251: 97 pgs: 97 active+clean; 141 MiB data, 346 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.5 MiB/s wr, 715 op/s 2024-09-20T18:27:29.914 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:29.914 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:29.914 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:29 smithi012 ceph-mon[23745]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 2024-09-20T18:27:30.012 INFO:teuthology.orchestra.run.smithi012.stdout: 5 8461 15.01 MB/sec warmup 21 sec latency 44.931 ms 2024-09-20T18:27:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:30.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: pgmap v251: 97 pgs: 97 active+clean; 141 MiB data, 346 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 8.5 MiB/s wr, 715 op/s 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:27:30.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:29 smithi174 ceph-mon[25682]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 2024-09-20T18:27:30.446 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-20T18:27:30.446 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:27:30.447 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-20T18:27:31.013 INFO:teuthology.orchestra.run.smithi012.stdout: 5 8883 14.67 MB/sec warmup 22 sec latency 56.352 ms 2024-09-20T18:27:31.965 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:31 smithi012 ceph-mon[23745]: pgmap v252: 97 pgs: 97 active+clean; 139 MiB data, 347 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 565 op/s 2024-09-20T18:27:32.013 INFO:teuthology.orchestra.run.smithi012.stdout: 5 9260 14.24 MB/sec warmup 23 sec latency 51.547 ms 2024-09-20T18:27:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:31 smithi174 ceph-mon[25682]: pgmap v252: 97 pgs: 97 active+clean; 139 MiB data, 347 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 6.4 MiB/s wr, 565 op/s 2024-09-20T18:27:33.013 INFO:teuthology.orchestra.run.smithi012.stdout: 5 9510 13.85 MB/sec warmup 24 sec latency 43.569 ms 2024-09-20T18:27:33.092 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (5m) 26s ago 7m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (6m) 26s ago 6m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (6m) 29s ago 6m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (5m) 26s ago 6m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (58s) 26s ago 58s 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (56s) 29s ago 55s 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (8m) 26s ago 8m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:27:33.890 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (6m) 29s ago 6m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (8m) 26s ago 8m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (6m) 29s ago 6m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (37s) 26s ago 37s 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (32s) 29s ago 31s 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (6m) 26s ago 6m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 29s ago 6m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (5m) 26s ago 5m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (4m) 26s ago 4m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (4m) 26s ago 4m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (4m) 26s ago 4m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (3m) 29s ago 3m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (3m) 29s ago 3m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (2m) 29s ago 2m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (2m) 29s ago 2m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:27:33.891 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (5m) 26s ago 5m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:27:34.021 INFO:teuthology.orchestra.run.smithi012.stdout: 5 9898 13.98 MB/sec warmup 25 sec latency 32.789 ms 2024-09-20T18:27:34.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:34 smithi012 ceph-mon[23745]: pgmap v253: 97 pgs: 97 active+clean; 139 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 727 op/s 2024-09-20T18:27:34.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:34 smithi012 ceph-mon[23745]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:34.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:34 smithi012 ceph-mon[23745]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:34.396 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:27:34.396 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:27:34.396 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:27:34.396 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:27:34.397 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:27:34.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:34 smithi174 ceph-mon[25682]: pgmap v253: 97 pgs: 97 active+clean; 139 MiB data, 403 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 727 op/s 2024-09-20T18:27:34.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:34 smithi174 ceph-mon[25682]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:34.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:34 smithi174 ceph-mon[25682]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:34.837 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:27:34.838 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:27:34.838 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:27:34.838 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:27:34.838 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:27:34.838 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:27:34.838 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:27:35.021 INFO:teuthology.orchestra.run.smithi012.stdout: 5 10458 14.39 MB/sec warmup 26 sec latency 53.648 ms 2024-09-20T18:27:35.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:35 smithi012 ceph-mon[23745]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:35.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:35 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1630932851' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:27:35.429 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:27:35.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:35 smithi174 ceph-mon[25682]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:35.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:35 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1630932851' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:27:36.021 INFO:teuthology.orchestra.run.smithi012.stdout: 5 10900 14.38 MB/sec warmup 27 sec latency 40.291 ms 2024-09-20T18:27:36.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:36 smithi012 ceph-mon[23745]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:36.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:36 smithi012 ceph-mon[23745]: pgmap v254: 97 pgs: 97 active+clean; 138 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 507 op/s 2024-09-20T18:27:36.090 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:36 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2239566462' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:27:36.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:36 smithi174 ceph-mon[25682]: from='client.14624 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:27:36.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:36 smithi174 ceph-mon[25682]: pgmap v254: 97 pgs: 97 active+clean; 138 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 507 op/s 2024-09-20T18:27:36.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:36 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2239566462' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:27:37.021 INFO:teuthology.orchestra.run.smithi012.stdout: 5 11247 14.01 MB/sec warmup 28 sec latency 49.134 ms 2024-09-20T18:27:38.021 INFO:teuthology.orchestra.run.smithi012.stdout: 5 11560 13.65 MB/sec warmup 29 sec latency 61.471 ms 2024-09-20T18:27:38.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:38 smithi012 ceph-mon[23745]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 478 op/s 2024-09-20T18:27:38.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:38 smithi174 ceph-mon[25682]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 435 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 478 op/s 2024-09-20T18:27:39.021 INFO:teuthology.orchestra.run.smithi012.stdout: 5 11841 13.43 MB/sec warmup 30 sec latency 50.473 ms 2024-09-20T18:27:40.030 INFO:teuthology.orchestra.run.smithi012.stdout: 5 12218 13.28 MB/sec warmup 31 sec latency 48.037 ms 2024-09-20T18:27:40.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:40 smithi012 ceph-mon[23745]: pgmap v256: 97 pgs: 97 active+clean; 156 MiB data, 432 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.1 MiB/s wr, 547 op/s 2024-09-20T18:27:40.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:40 smithi174 ceph-mon[25682]: pgmap v256: 97 pgs: 97 active+clean; 156 MiB data, 432 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.1 MiB/s wr, 547 op/s 2024-09-20T18:27:41.030 INFO:teuthology.orchestra.run.smithi012.stdout: 5 12674 13.13 MB/sec warmup 32 sec latency 58.354 ms 2024-09-20T18:27:42.011 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:41 smithi012 ceph-mon[23745]: pgmap v257: 97 pgs: 97 active+clean; 154 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-09-20T18:27:42.030 INFO:teuthology.orchestra.run.smithi012.stdout: 5 13018 12.87 MB/sec warmup 33 sec latency 70.222 ms 2024-09-20T18:27:42.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:41 smithi174 ceph-mon[25682]: pgmap v257: 97 pgs: 97 active+clean; 154 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 484 op/s 2024-09-20T18:27:43.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 13497 13.10 MB/sec warmup 34 sec latency 40.433 ms 2024-09-20T18:27:44.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 14058 13.38 MB/sec warmup 35 sec latency 41.070 ms 2024-09-20T18:27:44.298 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:44 smithi012 ceph-mon[23745]: pgmap v258: 97 pgs: 97 active+clean; 151 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 651 op/s 2024-09-20T18:27:44.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:44 smithi174 ceph-mon[25682]: pgmap v258: 97 pgs: 97 active+clean; 151 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 651 op/s 2024-09-20T18:27:45.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 14426 13.42 MB/sec warmup 36 sec latency 40.577 ms 2024-09-20T18:27:46.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 15033 13.23 MB/sec warmup 37 sec latency 42.641 ms 2024-09-20T18:27:46.066 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:46 smithi012 ceph-mon[23745]: pgmap v259: 97 pgs: 97 active+clean; 153 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 487 op/s 2024-09-20T18:27:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:46 smithi174 ceph-mon[25682]: pgmap v259: 97 pgs: 97 active+clean; 153 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 487 op/s 2024-09-20T18:27:47.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 15278 13.02 MB/sec warmup 38 sec latency 98.233 ms 2024-09-20T18:27:48.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 15579 12.98 MB/sec warmup 39 sec latency 63.420 ms 2024-09-20T18:27:48.194 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:48 smithi012 ceph-mon[23745]: pgmap v260: 97 pgs: 97 active+clean; 155 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.5 MiB/s wr, 513 op/s 2024-09-20T18:27:48.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:48 smithi174 ceph-mon[25682]: pgmap v260: 97 pgs: 97 active+clean; 155 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.5 MiB/s wr, 513 op/s 2024-09-20T18:27:49.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 16003 12.85 MB/sec warmup 40 sec latency 48.217 ms 2024-09-20T18:27:50.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 16409 12.66 MB/sec warmup 41 sec latency 52.392 ms 2024-09-20T18:27:50.233 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:50 smithi012 ceph-mon[23745]: pgmap v261: 97 pgs: 97 active+clean; 161 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 645 op/s 2024-09-20T18:27:50.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:50 smithi174 ceph-mon[25682]: pgmap v261: 97 pgs: 97 active+clean; 161 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 645 op/s 2024-09-20T18:27:51.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 16959 12.82 MB/sec warmup 42 sec latency 43.041 ms 2024-09-20T18:27:52.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:51 smithi012 ceph-mon[23745]: pgmap v262: 97 pgs: 97 active+clean; 160 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 498 op/s 2024-09-20T18:27:52.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 17568 13.03 MB/sec warmup 43 sec latency 64.386 ms 2024-09-20T18:27:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:51 smithi174 ceph-mon[25682]: pgmap v262: 97 pgs: 97 active+clean; 160 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 498 op/s 2024-09-20T18:27:53.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 17999 13.18 MB/sec warmup 44 sec latency 35.998 ms 2024-09-20T18:27:54.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 18311 12.96 MB/sec warmup 45 sec latency 74.418 ms 2024-09-20T18:27:54.295 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:54 smithi012 ceph-mon[23745]: pgmap v263: 97 pgs: 97 active+clean; 167 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 627 op/s 2024-09-20T18:27:54.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:54 smithi174 ceph-mon[25682]: pgmap v263: 97 pgs: 97 active+clean; 167 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 627 op/s 2024-09-20T18:27:55.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 18767 12.82 MB/sec warmup 46 sec latency 64.912 ms 2024-09-20T18:27:56.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 19348 12.89 MB/sec warmup 47 sec latency 54.359 ms 2024-09-20T18:27:56.308 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:56 smithi012 ceph-mon[23745]: pgmap v264: 97 pgs: 97 active+clean; 175 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 567 op/s 2024-09-20T18:27:56.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:56 smithi174 ceph-mon[25682]: pgmap v264: 97 pgs: 97 active+clean; 175 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 567 op/s 2024-09-20T18:27:57.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 20071 12.83 MB/sec warmup 48 sec latency 33.674 ms 2024-09-20T18:27:58.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 20562 12.99 MB/sec warmup 49 sec latency 43.497 ms 2024-09-20T18:27:58.240 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:58 smithi012 ceph-mon[23745]: pgmap v265: 97 pgs: 97 active+clean; 175 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 574 op/s 2024-09-20T18:27:58.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:58 smithi174 ceph-mon[25682]: pgmap v265: 97 pgs: 97 active+clean; 175 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.5 MiB/s wr, 574 op/s 2024-09-20T18:27:59.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 21137 13.13 MB/sec warmup 50 sec latency 51.177 ms 2024-09-20T18:27:59.258 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-20T18:27:59.259 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-20T18:27:59.259 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2024-09-20T18:27:59.259 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-20T18:27:59.259 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:27:59.259 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:27:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:27:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]: dispatch 2024-09-20T18:27:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-20T18:27:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2024-09-20T18:27:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-20T18:27:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:27:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:27:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:28:00.038 INFO:teuthology.orchestra.run.smithi012.stdout: 5 21510 13.23 MB/sec warmup 51 sec latency 56.337 ms 2024-09-20T18:28:00.302 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:00 smithi012 ceph-mon[23745]: pgmap v266: 97 pgs: 97 active+clean; 182 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 762 op/s 2024-09-20T18:28:00.302 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-20T18:28:00.302 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-20T18:28:00.303 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2024-09-20T18:28:00.303 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:00 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-20T18:28:00.303 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:00 smithi012 ceph-mon[23745]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T18:28:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:00 smithi174 ceph-mon[25682]: pgmap v266: 97 pgs: 97 active+clean; 182 MiB data, 683 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 762 op/s 2024-09-20T18:28:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.5", "id": [7, 2]}]': finished 2024-09-20T18:28:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-20T18:28:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2024-09-20T18:28:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:00 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-20T18:28:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:00 smithi174 ceph-mon[25682]: osdmap e69: 8 total, 8 up, 8 in 2024-09-20T18:28:01.041 INFO:teuthology.orchestra.run.smithi012.stdout: 5 21801 13.04 MB/sec warmup 52 sec latency 64.458 ms 2024-09-20T18:28:01.312 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:01 smithi012 ceph-mon[23745]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T18:28:01.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:01 smithi174 ceph-mon[25682]: osdmap e70: 8 total, 8 up, 8 in 2024-09-20T18:28:02.041 INFO:teuthology.orchestra.run.smithi012.stdout: 5 22094 12.84 MB/sec warmup 53 sec latency 80.291 ms 2024-09-20T18:28:02.337 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:02 smithi012 ceph-mon[23745]: pgmap v269: 97 pgs: 97 active+clean; 179 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 631 op/s 2024-09-20T18:28:02.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:02 smithi174 ceph-mon[25682]: pgmap v269: 97 pgs: 97 active+clean; 179 MiB data, 707 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 631 op/s 2024-09-20T18:28:03.041 INFO:teuthology.orchestra.run.smithi012.stdout: 5 22479 12.78 MB/sec warmup 54 sec latency 52.178 ms 2024-09-20T18:28:04.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 22871 12.75 MB/sec warmup 55 sec latency 81.805 ms 2024-09-20T18:28:04.120 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:04 smithi012 ceph-mon[23745]: pgmap v270: 97 pgs: 97 active+clean; 191 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 733 op/s 2024-09-20T18:28:04.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:04 smithi174 ceph-mon[25682]: pgmap v270: 97 pgs: 97 active+clean; 191 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 733 op/s 2024-09-20T18:28:05.041 INFO:teuthology.orchestra.run.smithi012.stdout: 5 23350 12.63 MB/sec warmup 56 sec latency 81.550 ms 2024-09-20T18:28:05.845 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:28:06.041 INFO:teuthology.orchestra.run.smithi012.stdout: 5 23682 12.51 MB/sec warmup 57 sec latency 70.802 ms 2024-09-20T18:28:06.144 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:06 smithi012 ceph-mon[23745]: pgmap v271: 97 pgs: 97 active+clean; 190 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 728 op/s 2024-09-20T18:28:06.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:06 smithi174 ceph-mon[25682]: pgmap v271: 97 pgs: 97 active+clean; 190 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 728 op/s 2024-09-20T18:28:06.597 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:28:06.597 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (6m) 59s ago 7m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:28:06.597 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (7m) 59s ago 7m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (6m) 62s ago 6m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (5m) 59s ago 7m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (91s) 59s ago 90s 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (88s) 62s ago 88s 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (8m) 59s ago 8m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (6m) 62s ago 6m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (8m) 59s ago 8m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (6m) 62s ago 6m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (70s) 59s ago 69s 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (64s) 62s ago 64s 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (7m) 59s ago 7m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 62s ago 6m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:28:06.598 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (6m) 59s ago 6m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (5m) 59s ago 5m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (5m) 59s ago 5m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (4m) 59s ago 4m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (4m) 62s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (3m) 62s ago 3m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (3m) 62s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (3m) 62s ago 3m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:28:06.599 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (6m) 59s ago 6m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:28:07.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 24144 12.62 MB/sec warmup 58 sec latency 40.596 ms 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:28:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:28:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:28:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:28:07.068 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:28:07.068 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:28:07.163 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:07 smithi012 ceph-mon[23745]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:07.163 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:07 smithi012 ceph-mon[23745]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:07.163 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:07 smithi012 ceph-mon[23745]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:07.163 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:07 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/48985911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:28:07.438 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:28:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:28:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:28:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:28:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:28:07.439 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:28:07.439 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:28:07.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:07 smithi174 ceph-mon[25682]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:07.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:07 smithi174 ceph-mon[25682]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:07.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:07 smithi174 ceph-mon[25682]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:07.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:07 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/48985911' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:28:08.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 24707 12.84 MB/sec warmup 59 sec latency 41.986 ms 2024-09-20T18:28:08.083 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:28:08.176 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:08 smithi012 ceph-mon[23745]: pgmap v272: 97 pgs: 97 active+clean; 193 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 6.1 MiB/s wr, 579 op/s 2024-09-20T18:28:08.176 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:08 smithi012 ceph-mon[23745]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:08.177 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:08 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/702303703' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:28:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:08 smithi174 ceph-mon[25682]: pgmap v272: 97 pgs: 97 active+clean; 193 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 6.1 MiB/s wr, 579 op/s 2024-09-20T18:28:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:08 smithi174 ceph-mon[25682]: from='client.14648 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:08 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/702303703' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:28:09.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 25322 12.90 MB/sec warmup 60 sec latency 32.318 ms 2024-09-20T18:28:10.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 26073 12.88 MB/sec warmup 61 sec latency 33.858 ms 2024-09-20T18:28:10.192 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:10 smithi012 ceph-mon[23745]: pgmap v273: 97 pgs: 97 active+clean; 191 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 530 op/s 2024-09-20T18:28:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:10 smithi174 ceph-mon[25682]: pgmap v273: 97 pgs: 97 active+clean; 191 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.9 MiB/s wr, 530 op/s 2024-09-20T18:28:11.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 27069 13.01 MB/sec warmup 62 sec latency 14.127 ms 2024-09-20T18:28:11.959 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:11 smithi012 ceph-mon[23745]: pgmap v274: 97 pgs: 97 active+clean; 201 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.8 MiB/s wr, 658 op/s 2024-09-20T18:28:12.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 28040 13.37 MB/sec warmup 63 sec latency 50.438 ms 2024-09-20T18:28:12.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:11 smithi174 ceph-mon[25682]: pgmap v274: 97 pgs: 97 active+clean; 201 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.8 MiB/s wr, 658 op/s 2024-09-20T18:28:13.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 28846 13.59 MB/sec warmup 64 sec latency 10.333 ms 2024-09-20T18:28:14.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 29546 13.55 MB/sec warmup 65 sec latency 24.623 ms 2024-09-20T18:28:14.223 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:14 smithi012 ceph-mon[23745]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 825 op/s 2024-09-20T18:28:14.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:14 smithi174 ceph-mon[25682]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 825 op/s 2024-09-20T18:28:15.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 30395 13.61 MB/sec warmup 66 sec latency 24.908 ms 2024-09-20T18:28:16.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 31358 13.86 MB/sec warmup 67 sec latency 22.687 ms 2024-09-20T18:28:16.302 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:16 smithi012 ceph-mon[23745]: pgmap v276: 97 pgs: 97 active+clean; 210 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 681 op/s 2024-09-20T18:28:16.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:16 smithi174 ceph-mon[25682]: pgmap v276: 97 pgs: 97 active+clean; 210 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 681 op/s 2024-09-20T18:28:17.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 32285 14.16 MB/sec warmup 68 sec latency 14.127 ms 2024-09-20T18:28:18.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 32895 14.05 MB/sec warmup 69 sec latency 65.125 ms 2024-09-20T18:28:18.311 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:18 smithi012 ceph-mon[23745]: pgmap v277: 97 pgs: 97 active+clean; 211 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.4 MiB/s wr, 751 op/s 2024-09-20T18:28:18.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:18 smithi174 ceph-mon[25682]: pgmap v277: 97 pgs: 97 active+clean; 211 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 8.4 MiB/s wr, 751 op/s 2024-09-20T18:28:19.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 33767 14.18 MB/sec warmup 70 sec latency 29.254 ms 2024-09-20T18:28:20.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 34451 14.14 MB/sec warmup 71 sec latency 76.336 ms 2024-09-20T18:28:20.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:20 smithi012 ceph-mon[23745]: pgmap v278: 97 pgs: 97 active+clean; 227 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 985 op/s 2024-09-20T18:28:20.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:20 smithi174 ceph-mon[25682]: pgmap v278: 97 pgs: 97 active+clean; 227 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 11 MiB/s wr, 985 op/s 2024-09-20T18:28:21.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 34829 14.18 MB/sec warmup 72 sec latency 49.005 ms 2024-09-20T18:28:22.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 35266 14.22 MB/sec warmup 73 sec latency 42.253 ms 2024-09-20T18:28:22.105 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:21 smithi012 ceph-mon[23745]: pgmap v279: 97 pgs: 97 active+clean; 226 MiB data, 1000 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 974 op/s 2024-09-20T18:28:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:21 smithi174 ceph-mon[25682]: pgmap v279: 97 pgs: 97 active+clean; 226 MiB data, 1000 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 11 MiB/s wr, 974 op/s 2024-09-20T18:28:23.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 35622 14.21 MB/sec warmup 74 sec latency 95.993 ms 2024-09-20T18:28:24.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 35937 14.14 MB/sec warmup 75 sec latency 65.164 ms 2024-09-20T18:28:24.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:24 smithi012 ceph-mon[23745]: pgmap v280: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-09-20T18:28:24.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:24 smithi174 ceph-mon[25682]: pgmap v280: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 12 MiB/s wr, 1.08k op/s 2024-09-20T18:28:25.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 36499 14.04 MB/sec warmup 76 sec latency 50.557 ms 2024-09-20T18:28:26.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 36845 13.98 MB/sec warmup 77 sec latency 65.713 ms 2024-09-20T18:28:26.175 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:26 smithi012 ceph-mon[23745]: pgmap v281: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 871 op/s 2024-09-20T18:28:26.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:26 smithi174 ceph-mon[25682]: pgmap v281: 97 pgs: 97 active+clean; 235 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.8 MiB/s wr, 871 op/s 2024-09-20T18:28:27.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 37195 13.90 MB/sec warmup 78 sec latency 76.664 ms 2024-09-20T18:28:28.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 37654 13.81 MB/sec warmup 79 sec latency 53.412 ms 2024-09-20T18:28:28.242 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:28 smithi012 ceph-mon[23745]: pgmap v282: 97 pgs: 97 active+clean; 234 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 848 op/s 2024-09-20T18:28:28.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:28 smithi174 ceph-mon[25682]: pgmap v282: 97 pgs: 97 active+clean; 234 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 848 op/s 2024-09-20T18:28:29.042 INFO:teuthology.orchestra.run.smithi012.stdout: 5 37976 13.75 MB/sec warmup 80 sec latency 79.057 ms 2024-09-20T18:28:30.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 38666 13.91 MB/sec warmup 81 sec latency 91.303 ms 2024-09-20T18:28:30.259 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:30 smithi012 ceph-mon[23745]: pgmap v283: 97 pgs: 97 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 903 op/s 2024-09-20T18:28:30.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:30 smithi174 ceph-mon[25682]: pgmap v283: 97 pgs: 97 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.1 MiB/s wr, 903 op/s 2024-09-20T18:28:31.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 39250 14.02 MB/sec warmup 82 sec latency 54.867 ms 2024-09-20T18:28:32.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 39411 13.91 MB/sec warmup 83 sec latency 161.351 ms 2024-09-20T18:28:32.055 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:31 smithi012 ceph-mon[23745]: pgmap v284: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 648 op/s 2024-09-20T18:28:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:31 smithi174 ceph-mon[25682]: pgmap v284: 97 pgs: 97 active+clean; 238 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 648 op/s 2024-09-20T18:28:33.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 39553 13.77 MB/sec warmup 84 sec latency 165.081 ms 2024-09-20T18:28:34.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 39767 13.63 MB/sec warmup 85 sec latency 212.192 ms 2024-09-20T18:28:34.104 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:34 smithi012 ceph-mon[23745]: pgmap v285: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 752 op/s 2024-09-20T18:28:34.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:34 smithi174 ceph-mon[25682]: pgmap v285: 97 pgs: 97 active+clean; 248 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 752 op/s 2024-09-20T18:28:35.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 40002 13.51 MB/sec warmup 86 sec latency 96.018 ms 2024-09-20T18:28:36.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 40294 13.46 MB/sec warmup 87 sec latency 101.704 ms 2024-09-20T18:28:36.144 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:36 smithi012 ceph-mon[23745]: pgmap v286: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 559 op/s 2024-09-20T18:28:36.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:36 smithi174 ceph-mon[25682]: pgmap v286: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 559 op/s 2024-09-20T18:28:37.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 40773 13.42 MB/sec warmup 88 sec latency 42.846 ms 2024-09-20T18:28:38.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 41249 13.35 MB/sec warmup 89 sec latency 46.001 ms 2024-09-20T18:28:38.193 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:38 smithi012 ceph-mon[23745]: pgmap v287: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.6 MiB/s wr, 541 op/s 2024-09-20T18:28:38.490 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:28:38.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:38 smithi174 ceph-mon[25682]: pgmap v287: 97 pgs: 97 active+clean; 249 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.6 MiB/s wr, 541 op/s 2024-09-20T18:28:39.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 41560 13.29 MB/sec warmup 90 sec latency 54.239 ms 2024-09-20T18:28:39.201 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:39 smithi012 ceph-mon[23745]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:39.305 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:28:39.305 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (6m) 92s ago 8m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:28:39.305 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (8m) 92s ago 8m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:28:39.305 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (7m) 94s ago 7m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:28:39.305 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (6m) 92s ago 7m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:28:39.305 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (2m) 92s ago 2m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:28:39.305 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (2m) 94s ago 2m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (9m) 92s ago 9m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (7m) 94s ago 7m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (9m) 92s ago 9m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (7m) 94s ago 7m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (102s) 92s ago 102s 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (97s) 94s ago 97s 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (7m) 92s ago 7m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 94s ago 7m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (6m) 92s ago 6m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (6m) 92s ago 6m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (5m) 92s ago 5m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (5m) 92s ago 5m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (4m) 94s ago 4m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:28:39.306 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (4m) 94s ago 4m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:28:39.307 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (3m) 94s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:28:39.307 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (3m) 94s ago 3m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:28:39.307 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (7m) 92s ago 7m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:28:39.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:39 smithi174 ceph-mon[25682]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:39.921 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:28:39.922 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:28:39.923 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:28:39.923 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:28:39.923 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:28:40.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 41845 13.28 MB/sec warmup 91 sec latency 45.331 ms 2024-09-20T18:28:40.258 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:40 smithi012 ceph-mon[23745]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:40.258 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:40 smithi012 ceph-mon[23745]: pgmap v288: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 644 op/s 2024-09-20T18:28:40.258 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:40 smithi012 ceph-mon[23745]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:40.258 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:40 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/385965382' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:28:40.360 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:28:40.361 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:28:40.361 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:28:40.361 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:28:40.361 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:28:40.361 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:28:40.361 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:28:40.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:40 smithi174 ceph-mon[25682]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:40.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:40 smithi174 ceph-mon[25682]: pgmap v288: 97 pgs: 97 active+clean; 247 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 644 op/s 2024-09-20T18:28:40.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:40 smithi174 ceph-mon[25682]: from='client.14664 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:40.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:40 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/385965382' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:28:40.980 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:28:41.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 42332 13.34 MB/sec warmup 92 sec latency 33.525 ms 2024-09-20T18:28:41.336 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:41 smithi012 ceph-mon[23745]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:41.336 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:41 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/4142085946' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:28:41.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:41 smithi174 ceph-mon[25682]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:28:41.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:41 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/4142085946' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:28:42.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 42582 13.32 MB/sec warmup 93 sec latency 79.410 ms 2024-09-20T18:28:42.298 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:42 smithi012 ceph-mon[23745]: pgmap v289: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-09-20T18:28:42.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:42 smithi174 ceph-mon[25682]: pgmap v289: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 513 op/s 2024-09-20T18:28:43.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 42879 13.29 MB/sec warmup 94 sec latency 61.241 ms 2024-09-20T18:28:44.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 43076 13.18 MB/sec warmup 95 sec latency 71.718 ms 2024-09-20T18:28:44.337 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:44 smithi012 ceph-mon[23745]: pgmap v290: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-20T18:28:44.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:44 smithi174 ceph-mon[25682]: pgmap v290: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-20T18:28:45.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 43510 13.09 MB/sec warmup 96 sec latency 41.517 ms 2024-09-20T18:28:46.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 43893 13.05 MB/sec warmup 97 sec latency 43.377 ms 2024-09-20T18:28:46.370 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:46 smithi012 ceph-mon[23745]: pgmap v291: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 428 op/s 2024-09-20T18:28:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:46 smithi174 ceph-mon[25682]: pgmap v291: 97 pgs: 97 active+clean; 263 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 428 op/s 2024-09-20T18:28:47.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 44373 13.06 MB/sec warmup 98 sec latency 94.242 ms 2024-09-20T18:28:48.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 45006 13.02 MB/sec warmup 99 sec latency 84.221 ms 2024-09-20T18:28:48.174 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:48 smithi012 ceph-mon[23745]: pgmap v292: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 373 op/s 2024-09-20T18:28:48.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:48 smithi174 ceph-mon[25682]: pgmap v292: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 373 op/s 2024-09-20T18:28:49.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 45387 13.04 MB/sec warmup 100 sec latency 43.717 ms 2024-09-20T18:28:50.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 45965 13.15 MB/sec warmup 101 sec latency 32.661 ms 2024-09-20T18:28:50.202 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:50 smithi012 ceph-mon[23745]: pgmap v293: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 552 op/s 2024-09-20T18:28:50.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:50 smithi174 ceph-mon[25682]: pgmap v293: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 552 op/s 2024-09-20T18:28:51.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 46409 13.19 MB/sec warmup 102 sec latency 41.678 ms 2024-09-20T18:28:51.988 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:51 smithi012 ceph-mon[23745]: pgmap v294: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 543 op/s 2024-09-20T18:28:52.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 46711 13.09 MB/sec warmup 103 sec latency 48.580 ms 2024-09-20T18:28:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:51 smithi174 ceph-mon[25682]: pgmap v294: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.2 MiB/s wr, 543 op/s 2024-09-20T18:28:53.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 47299 13.06 MB/sec warmup 104 sec latency 52.820 ms 2024-09-20T18:28:54.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 47963 13.10 MB/sec warmup 105 sec latency 34.175 ms 2024-09-20T18:28:54.252 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:54 smithi012 ceph-mon[23745]: pgmap v295: 97 pgs: 97 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.5 MiB/s wr, 646 op/s 2024-09-20T18:28:54.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:54 smithi174 ceph-mon[25682]: pgmap v295: 97 pgs: 97 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.5 MiB/s wr, 646 op/s 2024-09-20T18:28:55.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 48609 13.08 MB/sec warmup 106 sec latency 54.719 ms 2024-09-20T18:28:56.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 49097 13.15 MB/sec warmup 107 sec latency 83.561 ms 2024-09-20T18:28:56.291 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:56 smithi012 ceph-mon[23745]: pgmap v296: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 519 op/s 2024-09-20T18:28:56.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:56 smithi174 ceph-mon[25682]: pgmap v296: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 519 op/s 2024-09-20T18:28:57.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 49659 13.22 MB/sec warmup 108 sec latency 32.418 ms 2024-09-20T18:28:58.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 50169 13.24 MB/sec warmup 109 sec latency 51.785 ms 2024-09-20T18:28:58.316 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:58 smithi012 ceph-mon[23745]: pgmap v297: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 541 op/s 2024-09-20T18:28:58.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:58 smithi174 ceph-mon[25682]: pgmap v297: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 541 op/s 2024-09-20T18:28:59.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 50524 13.16 MB/sec warmup 110 sec latency 44.117 ms 2024-09-20T18:28:59.323 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:28:59.323 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:28:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:28:59.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:28:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:28:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:29:00.043 INFO:teuthology.orchestra.run.smithi012.stdout: 5 51205 13.18 MB/sec warmup 111 sec latency 31.208 ms 2024-09-20T18:29:00.308 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:00 smithi012 ceph-mon[23745]: pgmap v298: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 771 op/s 2024-09-20T18:29:00.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:00 smithi174 ceph-mon[25682]: pgmap v298: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.7 MiB/s wr, 771 op/s 2024-09-20T18:29:01.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 51955 13.20 MB/sec warmup 112 sec latency 29.810 ms 2024-09-20T18:29:02.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 52705 13.32 MB/sec warmup 113 sec latency 65.161 ms 2024-09-20T18:29:02.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:01 smithi012 ceph-mon[23745]: pgmap v299: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.5 MiB/s wr, 675 op/s 2024-09-20T18:29:02.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:01 smithi174 ceph-mon[25682]: pgmap v299: 97 pgs: 97 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.5 MiB/s wr, 675 op/s 2024-09-20T18:29:03.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 53436 13.44 MB/sec warmup 114 sec latency 15.797 ms 2024-09-20T18:29:04.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 53996 13.41 MB/sec warmup 115 sec latency 22.032 ms 2024-09-20T18:29:04.101 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:04 smithi012 ceph-mon[23745]: pgmap v300: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 713 op/s 2024-09-20T18:29:04.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:04 smithi174 ceph-mon[25682]: pgmap v300: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 713 op/s 2024-09-20T18:29:05.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 54841 13.44 MB/sec warmup 116 sec latency 16.814 ms 2024-09-20T18:29:06.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 55649 13.49 MB/sec warmup 117 sec latency 25.726 ms 2024-09-20T18:29:06.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:06 smithi012 ceph-mon[23745]: pgmap v301: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 737 op/s 2024-09-20T18:29:06.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:06 smithi174 ceph-mon[25682]: pgmap v301: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 737 op/s 2024-09-20T18:29:07.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 56318 13.60 MB/sec warmup 118 sec latency 200.976 ms 2024-09-20T18:29:08.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 57025 13.68 MB/sec warmup 119 sec latency 21.023 ms 2024-09-20T18:29:08.142 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:08 smithi012 ceph-mon[23745]: pgmap v302: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.6 MiB/s wr, 810 op/s 2024-09-20T18:29:08.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:08 smithi174 ceph-mon[25682]: pgmap v302: 97 pgs: 97 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.6 MiB/s wr, 810 op/s 2024-09-20T18:29:10.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 58021 9.11 MB/sec execute 1 sec latency 49.139 ms 2024-09-20T18:29:10.194 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:10 smithi012 ceph-mon[23745]: pgmap v303: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-20T18:29:10.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:10 smithi174 ceph-mon[25682]: pgmap v303: 97 pgs: 97 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 10 MiB/s wr, 1.01k op/s 2024-09-20T18:29:11.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 58411 8.31 MB/sec execute 2 sec latency 50.671 ms 2024-09-20T18:29:11.462 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:29:11.970 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:11 smithi012 ceph-mon[23745]: pgmap v304: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 855 op/s 2024-09-20T18:29:11.970 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:11 smithi012 ceph-mon[23745]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:12.046 INFO:teuthology.orchestra.run.smithi012.stdout: 5 58788 8.88 MB/sec execute 3 sec latency 78.711 ms 2024-09-20T18:29:12.255 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:29:12.255 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (7m) 2m ago 8m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:29:12.255 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (8m) 2m ago 8m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:29:12.255 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (8m) 2m ago 8m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:29:12.255 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (7m) 2m ago 8m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (2m) 2m ago 2m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (2m) 2m ago 2m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (8m) 2m ago 8m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (9m) 2m ago 10m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (7m) 2m ago 7m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (2m) 2m ago 2m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (2m) 2m ago 2m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (8m) 2m ago 8m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 2m ago 7m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:29:12.256 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (7m) 2m ago 7m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (6m) 2m ago 6m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (6m) 2m ago 6m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (5m) 2m ago 5m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (5m) 2m ago 5m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (4m) 2m ago 4m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (4m) 2m ago 4m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (4m) 2m ago 4m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:29:12.257 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (7m) 2m ago 7m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:29:12.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:11 smithi174 ceph-mon[25682]: pgmap v304: 97 pgs: 97 active+clean; 311 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 855 op/s 2024-09-20T18:29:12.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:11 smithi174 ceph-mon[25682]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:12.835 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:29:12.835 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:29:12.835 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:29:12.835 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:12.835 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:29:12.835 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:29:12.836 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:29:12.979 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:12 smithi012 ceph-mon[23745]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:12.980 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:12 smithi012 ceph-mon[23745]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:12.980 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:12 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2094947769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:29:13.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 59239 9.47 MB/sec execute 4 sec latency 82.827 ms 2024-09-20T18:29:13.251 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:29:13.251 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:29:13.251 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:29:13.251 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:29:13.251 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:29:13.251 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:29:13.251 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:29:13.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:12 smithi174 ceph-mon[25682]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:13.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:12 smithi174 ceph-mon[25682]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:13.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:12 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2094947769' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:29:13.704 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:29:14.008 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:13 smithi012 ceph-mon[23745]: pgmap v305: 97 pgs: 97 active+clean; 313 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.6 MiB/s wr, 957 op/s 2024-09-20T18:29:14.009 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:13 smithi012 ceph-mon[23745]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:14.009 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:13 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3716604052' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:29:14.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 59552 9.15 MB/sec execute 5 sec latency 58.147 ms 2024-09-20T18:29:14.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:13 smithi174 ceph-mon[25682]: pgmap v305: 97 pgs: 97 active+clean; 313 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 9.6 MiB/s wr, 957 op/s 2024-09-20T18:29:14.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:13 smithi174 ceph-mon[25682]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:14.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:13 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3716604052' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:29:15.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 59947 10.70 MB/sec execute 6 sec latency 41.576 ms 2024-09-20T18:29:16.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 60368 11.25 MB/sec execute 7 sec latency 60.340 ms 2024-09-20T18:29:16.284 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:16 smithi012 ceph-mon[23745]: pgmap v306: 97 pgs: 97 active+clean; 311 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 830 op/s 2024-09-20T18:29:16.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:16 smithi174 ceph-mon[25682]: pgmap v306: 97 pgs: 97 active+clean; 311 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 830 op/s 2024-09-20T18:29:17.044 INFO:teuthology.orchestra.run.smithi012.stdout: 5 60785 11.29 MB/sec execute 8 sec latency 33.001 ms 2024-09-20T18:29:18.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 61161 10.93 MB/sec execute 9 sec latency 39.291 ms 2024-09-20T18:29:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:18 smithi012 ceph-mon[23745]: pgmap v307: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 708 op/s 2024-09-20T18:29:18.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:18 smithi174 ceph-mon[25682]: pgmap v307: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 708 op/s 2024-09-20T18:29:19.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 61728 10.58 MB/sec execute 10 sec latency 65.061 ms 2024-09-20T18:29:20.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 62234 11.05 MB/sec execute 11 sec latency 32.336 ms 2024-09-20T18:29:20.339 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:20 smithi012 ceph-mon[23745]: pgmap v308: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.2 MiB/s wr, 805 op/s 2024-09-20T18:29:20.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:20 smithi174 ceph-mon[25682]: pgmap v308: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.2 MiB/s wr, 805 op/s 2024-09-20T18:29:21.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 62833 11.21 MB/sec execute 12 sec latency 44.698 ms 2024-09-20T18:29:22.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 63245 11.21 MB/sec execute 13 sec latency 44.657 ms 2024-09-20T18:29:22.141 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:21 smithi012 ceph-mon[23745]: pgmap v309: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 602 op/s 2024-09-20T18:29:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:21 smithi174 ceph-mon[25682]: pgmap v309: 97 pgs: 97 active+clean; 326 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 602 op/s 2024-09-20T18:29:23.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 63711 11.96 MB/sec execute 14 sec latency 73.269 ms 2024-09-20T18:29:24.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 64236 12.24 MB/sec execute 15 sec latency 44.454 ms 2024-09-20T18:29:24.183 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:24 smithi012 ceph-mon[23745]: pgmap v310: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 646 op/s 2024-09-20T18:29:24.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:24 smithi174 ceph-mon[25682]: pgmap v310: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 646 op/s 2024-09-20T18:29:25.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 64656 12.11 MB/sec execute 16 sec latency 64.748 ms 2024-09-20T18:29:26.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 65142 11.73 MB/sec execute 17 sec latency 79.164 ms 2024-09-20T18:29:26.131 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:26 smithi012 ceph-mon[23745]: pgmap v311: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.5 MiB/s wr, 638 op/s 2024-09-20T18:29:26.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:26 smithi174 ceph-mon[25682]: pgmap v311: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.5 MiB/s wr, 638 op/s 2024-09-20T18:29:27.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 65764 12.06 MB/sec execute 18 sec latency 21.742 ms 2024-09-20T18:29:28.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 66521 12.25 MB/sec execute 19 sec latency 45.825 ms 2024-09-20T18:29:28.144 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:28 smithi012 ceph-mon[23745]: pgmap v312: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.9 MiB/s wr, 656 op/s 2024-09-20T18:29:28.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:28 smithi174 ceph-mon[25682]: pgmap v312: 97 pgs: 97 active+clean; 336 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.9 MiB/s wr, 656 op/s 2024-09-20T18:29:29.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 66879 12.33 MB/sec execute 20 sec latency 38.161 ms 2024-09-20T18:29:30.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 67442 12.66 MB/sec execute 21 sec latency 41.043 ms 2024-09-20T18:29:30.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:30 smithi012 ceph-mon[23745]: pgmap v313: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 844 op/s 2024-09-20T18:29:30.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:30 smithi174 ceph-mon[25682]: pgmap v313: 97 pgs: 97 active+clean; 340 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.0 MiB/s wr, 844 op/s 2024-09-20T18:29:31.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 67844 12.74 MB/sec execute 22 sec latency 43.264 ms 2024-09-20T18:29:32.045 INFO:teuthology.orchestra.run.smithi012.stdout: 5 68158 12.58 MB/sec execute 23 sec latency 61.235 ms 2024-09-20T18:29:32.214 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:31 smithi012 ceph-mon[23745]: pgmap v314: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 673 op/s 2024-09-20T18:29:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:31 smithi174 ceph-mon[25682]: pgmap v314: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.5 MiB/s wr, 673 op/s 2024-09-20T18:29:33.054 INFO:teuthology.orchestra.run.smithi012.stdout: 5 68668 12.28 MB/sec execute 24 sec latency 33.368 ms 2024-09-20T18:29:34.054 INFO:teuthology.orchestra.run.smithi012.stdout: 5 69312 12.51 MB/sec execute 25 sec latency 22.416 ms 2024-09-20T18:29:34.245 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:34 smithi012 ceph-mon[23745]: pgmap v315: 97 pgs: 97 active+clean; 345 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 853 op/s 2024-09-20T18:29:34.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:34 smithi174 ceph-mon[25682]: pgmap v315: 97 pgs: 97 active+clean; 345 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 853 op/s 2024-09-20T18:29:35.054 INFO:teuthology.orchestra.run.smithi012.stdout: 5 70002 12.60 MB/sec execute 26 sec latency 33.552 ms 2024-09-20T18:29:36.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 70469 12.79 MB/sec execute 27 sec latency 29.109 ms 2024-09-20T18:29:36.277 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:36 smithi012 ceph-mon[23745]: pgmap v316: 97 pgs: 97 active+clean; 349 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.5 MiB/s wr, 809 op/s 2024-09-20T18:29:36.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:36 smithi174 ceph-mon[25682]: pgmap v316: 97 pgs: 97 active+clean; 349 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.5 MiB/s wr, 809 op/s 2024-09-20T18:29:37.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 71046 12.97 MB/sec execute 28 sec latency 27.903 ms 2024-09-20T18:29:38.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 71611 13.14 MB/sec execute 29 sec latency 23.848 ms 2024-09-20T18:29:38.293 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:38 smithi012 ceph-mon[23745]: pgmap v317: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.3 MiB/s wr, 753 op/s 2024-09-20T18:29:38.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:38 smithi174 ceph-mon[25682]: pgmap v317: 97 pgs: 97 active+clean; 349 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.3 MiB/s wr, 753 op/s 2024-09-20T18:29:39.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 72214 13.01 MB/sec execute 30 sec latency 31.765 ms 2024-09-20T18:29:40.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 72639 12.93 MB/sec execute 31 sec latency 63.556 ms 2024-09-20T18:29:40.389 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:40 smithi012 ceph-mon[23745]: pgmap v318: 97 pgs: 97 active+clean; 350 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 853 op/s 2024-09-20T18:29:40.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:40 smithi174 ceph-mon[25682]: pgmap v318: 97 pgs: 97 active+clean; 350 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 853 op/s 2024-09-20T18:29:41.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 73045 12.90 MB/sec execute 32 sec latency 65.325 ms 2024-09-20T18:29:42.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 73562 12.82 MB/sec execute 33 sec latency 50.688 ms 2024-09-20T18:29:42.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:41 smithi012 ceph-mon[23745]: pgmap v319: 97 pgs: 97 active+clean; 350 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.3 MiB/s wr, 685 op/s 2024-09-20T18:29:42.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:41 smithi174 ceph-mon[25682]: pgmap v319: 97 pgs: 97 active+clean; 350 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.3 MiB/s wr, 685 op/s 2024-09-20T18:29:43.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 74167 13.07 MB/sec execute 34 sec latency 19.846 ms 2024-09-20T18:29:44.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 74823 13.35 MB/sec execute 35 sec latency 25.665 ms 2024-09-20T18:29:44.199 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:44 smithi012 ceph-mon[23745]: pgmap v320: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 814 op/s 2024-09-20T18:29:44.261 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:29:44.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:44 smithi174 ceph-mon[25682]: pgmap v320: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 814 op/s 2024-09-20T18:29:45.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 75433 13.44 MB/sec execute 36 sec latency 24.261 ms 2024-09-20T18:29:45.203 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:45 smithi012 ceph-mon[23745]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:45.302 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:29:45.302 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (7m) 2m ago 9m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:29:45.302 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (9m) 2m ago 9m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:29:45.302 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (8m) 2m ago 8m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:29:45.302 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (7m) 2m ago 8m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:29:45.302 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (3m) 2m ago 3m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (3m) 2m ago 3m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (10m) 2m ago 10m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (8m) 2m ago 8m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (10m) 2m ago 10m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (8m) 2m ago 8m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (2m) 2m ago 2m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (2m) 2m ago 2m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (8m) 2m ago 8m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 2m ago 8m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (7m) 2m ago 7m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (7m) 2m ago 7m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (6m) 2m ago 6m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (6m) 2m ago 6m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (5m) 2m ago 5m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:29:45.303 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (5m) 2m ago 5m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:29:45.304 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (5m) 2m ago 5m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:29:45.304 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (4m) 2m ago 4m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:29:45.304 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (8m) 2m ago 8m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:29:45.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:45 smithi174 ceph-mon[25682]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:45.949 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:29:45.950 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:29:46.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 76011 13.21 MB/sec execute 37 sec latency 23.933 ms 2024-09-20T18:29:46.208 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:46 smithi012 ceph-mon[23745]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:46.208 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:46 smithi012 ceph-mon[23745]: pgmap v321: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 699 op/s 2024-09-20T18:29:46.208 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:46 smithi012 ceph-mon[23745]: from='client.24455 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:46.208 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:46 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2874499534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:29:46.508 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:29:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:29:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:29:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:29:46.508 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:29:46.509 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:29:46.509 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:29:46.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:46 smithi174 ceph-mon[25682]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:46 smithi174 ceph-mon[25682]: pgmap v321: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 699 op/s 2024-09-20T18:29:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:46 smithi174 ceph-mon[25682]: from='client.24455 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:46 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2874499534' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:29:47.063 INFO:teuthology.orchestra.run.smithi012.stdout: 5 76526 13.29 MB/sec execute 38 sec latency 35.887 ms 2024-09-20T18:29:47.137 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:29:47.206 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:47 smithi012 ceph-mon[23745]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:47.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:47 smithi174 ceph-mon[25682]: from='client.14718 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:29:48.065 INFO:teuthology.orchestra.run.smithi012.stdout: 5 77217 13.36 MB/sec execute 39 sec latency 35.084 ms 2024-09-20T18:29:48.221 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:48 smithi012 ceph-mon[23745]: pgmap v322: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.5 MiB/s wr, 702 op/s 2024-09-20T18:29:48.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:48 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2947209000' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:29:48.419 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:48 smithi174 ceph-mon[25682]: pgmap v322: 97 pgs: 97 active+clean; 337 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.5 MiB/s wr, 702 op/s 2024-09-20T18:29:48.420 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:48 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2947209000' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:29:49.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 77650 13.49 MB/sec execute 40 sec latency 53.815 ms 2024-09-20T18:29:50.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 78115 13.55 MB/sec execute 41 sec latency 32.760 ms 2024-09-20T18:29:50.269 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:50 smithi012 ceph-mon[23745]: pgmap v323: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 717 op/s 2024-09-20T18:29:50.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:50 smithi174 ceph-mon[25682]: pgmap v323: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.0 MiB/s wr, 717 op/s 2024-09-20T18:29:51.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 78540 13.51 MB/sec execute 42 sec latency 41.989 ms 2024-09-20T18:29:52.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 78747 13.31 MB/sec execute 43 sec latency 86.623 ms 2024-09-20T18:29:52.105 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:51 smithi012 ceph-mon[23745]: pgmap v324: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 700 op/s 2024-09-20T18:29:52.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:51 smithi174 ceph-mon[25682]: pgmap v324: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 700 op/s 2024-09-20T18:29:53.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 79140 13.19 MB/sec execute 44 sec latency 40.563 ms 2024-09-20T18:29:54.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 79601 12.99 MB/sec execute 45 sec latency 41.519 ms 2024-09-20T18:29:54.364 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:54 smithi012 ceph-mon[23745]: pgmap v325: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.4 MiB/s wr, 844 op/s 2024-09-20T18:29:54.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:54 smithi174 ceph-mon[25682]: pgmap v325: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 8.4 MiB/s wr, 844 op/s 2024-09-20T18:29:55.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 79836 12.88 MB/sec execute 46 sec latency 67.956 ms 2024-09-20T18:29:56.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 80254 12.92 MB/sec execute 47 sec latency 75.561 ms 2024-09-20T18:29:56.136 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:56 smithi012 ceph-mon[23745]: pgmap v326: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.5 MiB/s wr, 714 op/s 2024-09-20T18:29:56.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:56 smithi174 ceph-mon[25682]: pgmap v326: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.5 MiB/s wr, 714 op/s 2024-09-20T18:29:57.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 80750 12.81 MB/sec execute 48 sec latency 48.312 ms 2024-09-20T18:29:58.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 81110 12.90 MB/sec execute 49 sec latency 49.016 ms 2024-09-20T18:29:58.221 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:58 smithi012 ceph-mon[23745]: pgmap v327: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.9 MiB/s wr, 641 op/s 2024-09-20T18:29:58.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:58 smithi174 ceph-mon[25682]: pgmap v327: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.9 MiB/s wr, 641 op/s 2024-09-20T18:29:59.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 81516 12.93 MB/sec execute 50 sec latency 86.582 ms 2024-09-20T18:29:59.237 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:29:59.237 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:29:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:29:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:29:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:29:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:30:00.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 82101 13.00 MB/sec execute 51 sec latency 55.154 ms 2024-09-20T18:30:00.241 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:00 smithi012 ceph-mon[23745]: pgmap v328: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 692 op/s 2024-09-20T18:30:00.241 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:00 smithi012 ceph-mon[23745]: overall HEALTH_OK 2024-09-20T18:30:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:00 smithi174 ceph-mon[25682]: pgmap v328: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 692 op/s 2024-09-20T18:30:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:00 smithi174 ceph-mon[25682]: overall HEALTH_OK 2024-09-20T18:30:01.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 82506 12.93 MB/sec execute 52 sec latency 41.002 ms 2024-09-20T18:30:02.015 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:01 smithi012 ceph-mon[23745]: pgmap v329: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 648 op/s 2024-09-20T18:30:02.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 82889 12.80 MB/sec execute 53 sec latency 60.252 ms 2024-09-20T18:30:02.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:01 smithi174 ceph-mon[25682]: pgmap v329: 97 pgs: 97 active+clean; 334 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.3 MiB/s wr, 648 op/s 2024-09-20T18:30:03.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 83275 12.73 MB/sec execute 54 sec latency 52.473 ms 2024-09-20T18:30:04.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 83649 12.66 MB/sec execute 55 sec latency 43.319 ms 2024-09-20T18:30:04.311 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:04 smithi012 ceph-mon[23745]: pgmap v330: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 681 op/s 2024-09-20T18:30:04.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:04 smithi174 ceph-mon[25682]: pgmap v330: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 681 op/s 2024-09-20T18:30:05.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 84210 12.66 MB/sec execute 56 sec latency 29.510 ms 2024-09-20T18:30:06.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 84654 12.74 MB/sec execute 57 sec latency 43.011 ms 2024-09-20T18:30:06.338 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:06 smithi012 ceph-mon[23745]: pgmap v331: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 515 op/s 2024-09-20T18:30:06.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:06 smithi174 ceph-mon[25682]: pgmap v331: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 515 op/s 2024-09-20T18:30:07.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 84976 12.77 MB/sec execute 58 sec latency 40.683 ms 2024-09-20T18:30:08.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 85464 12.77 MB/sec execute 59 sec latency 55.286 ms 2024-09-20T18:30:08.359 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:08 smithi012 ceph-mon[23745]: pgmap v332: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 513 op/s 2024-09-20T18:30:08.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:08 smithi174 ceph-mon[25682]: pgmap v332: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 513 op/s 2024-09-20T18:30:09.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 85861 12.73 MB/sec execute 60 sec latency 24.020 ms 2024-09-20T18:30:10.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 86265 12.65 MB/sec execute 61 sec latency 54.052 ms 2024-09-20T18:30:10.144 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:10 smithi012 ceph-mon[23745]: pgmap v333: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 694 op/s 2024-09-20T18:30:10.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:10 smithi174 ceph-mon[25682]: pgmap v333: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 694 op/s 2024-09-20T18:30:11.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 86719 12.54 MB/sec execute 62 sec latency 42.327 ms 2024-09-20T18:30:12.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 87206 12.62 MB/sec execute 63 sec latency 36.511 ms 2024-09-20T18:30:12.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:11 smithi012 ceph-mon[23745]: pgmap v334: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.1 MiB/s wr, 554 op/s 2024-09-20T18:30:12.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:11 smithi174 ceph-mon[25682]: pgmap v334: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.1 MiB/s wr, 554 op/s 2024-09-20T18:30:13.066 INFO:teuthology.orchestra.run.smithi012.stdout: 5 87951 12.69 MB/sec execute 64 sec latency 23.265 ms 2024-09-20T18:30:14.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 88499 12.83 MB/sec execute 65 sec latency 20.298 ms 2024-09-20T18:30:14.278 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:14 smithi012 ceph-mon[23745]: pgmap v335: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 701 op/s 2024-09-20T18:30:14.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:14 smithi174 ceph-mon[25682]: pgmap v335: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 701 op/s 2024-09-20T18:30:15.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 89110 12.92 MB/sec execute 66 sec latency 31.691 ms 2024-09-20T18:30:16.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 89611 12.93 MB/sec execute 67 sec latency 21.890 ms 2024-09-20T18:30:16.293 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:16 smithi012 ceph-mon[23745]: pgmap v336: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 576 op/s 2024-09-20T18:30:16.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:16 smithi174 ceph-mon[25682]: pgmap v336: 97 pgs: 97 active+clean; 330 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.0 MiB/s wr, 576 op/s 2024-09-20T18:30:17.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 90386 12.88 MB/sec execute 68 sec latency 29.177 ms 2024-09-20T18:30:17.696 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:30:18.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 91031 13.00 MB/sec execute 69 sec latency 40.400 ms 2024-09-20T18:30:18.323 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:18 smithi012 ceph-mon[23745]: pgmap v337: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 585 op/s 2024-09-20T18:30:18.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:18 smithi174 ceph-mon[25682]: pgmap v337: 97 pgs: 97 active+clean; 333 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 585 op/s 2024-09-20T18:30:18.605 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:30:18.606 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (8m) 3m ago 9m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:30:18.606 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (9m) 3m ago 9m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:30:18.606 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (9m) 3m ago 9m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:30:18.606 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (8m) 3m ago 9m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:30:18.606 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (3m) 3m ago 3m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:30:18.606 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (3m) 3m ago 3m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:30:18.607 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (11m) 3m ago 11m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:30:18.607 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (9m) 3m ago 9m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:30:18.607 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (11m) 3m ago 11m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:30:18.607 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (9m) 3m ago 9m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:30:18.607 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (3m) 3m ago 3m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:30:18.607 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (3m) 3m ago 3m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:30:18.608 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (9m) 3m ago 9m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:30:18.608 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 3m ago 8m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:30:18.608 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (8m) 3m ago 8m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:30:18.608 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (7m) 3m ago 7m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:30:18.608 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (7m) 3m ago 7m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:30:18.608 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (6m) 3m ago 6m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:30:18.608 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (6m) 3m ago 6m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:30:18.609 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (6m) 3m ago 6m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:30:18.609 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (5m) 3m ago 5m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:30:18.609 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (5m) 3m ago 5m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:30:18.609 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (8m) 3m ago 8m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:30:19.057 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:30:19.058 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:19.059 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:30:19.059 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:30:19.059 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:30:19.059 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:30:19.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 91382 12.89 MB/sec execute 70 sec latency 58.161 ms 2024-09-20T18:30:19.357 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:19 smithi012 ceph-mon[23745]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:19.357 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:19 smithi012 ceph-mon[23745]: from='client.24473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:19.357 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:19 smithi012 ceph-mon[23745]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:19.357 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:19 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/775149979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:30:19.434 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:30:19.434 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:30:19.434 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:30:19.434 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:30:19.434 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:30:19.434 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:30:19.434 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:30:19.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:19 smithi174 ceph-mon[25682]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:19.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:19 smithi174 ceph-mon[25682]: from='client.24473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:19.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:19 smithi174 ceph-mon[25682]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:19.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:19 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/775149979' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:30:19.887 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:30:20.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 91617 12.85 MB/sec execute 71 sec latency 60.793 ms 2024-09-20T18:30:20.371 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:20 smithi012 ceph-mon[23745]: pgmap v338: 97 pgs: 97 active+clean; 346 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 850 op/s 2024-09-20T18:30:20.371 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:20 smithi012 ceph-mon[23745]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:20.371 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:20 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/530290266' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:30:20.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:20 smithi174 ceph-mon[25682]: pgmap v338: 97 pgs: 97 active+clean; 346 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 850 op/s 2024-09-20T18:30:20.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:20 smithi174 ceph-mon[25682]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:20.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:20 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/530290266' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:30:21.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 91891 12.84 MB/sec execute 72 sec latency 52.859 ms 2024-09-20T18:30:22.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 92153 12.78 MB/sec execute 73 sec latency 56.816 ms 2024-09-20T18:30:22.171 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:21 smithi012 ceph-mon[23745]: pgmap v339: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 665 op/s 2024-09-20T18:30:22.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:21 smithi174 ceph-mon[25682]: pgmap v339: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.0 MiB/s wr, 665 op/s 2024-09-20T18:30:23.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 92567 12.80 MB/sec execute 74 sec latency 72.832 ms 2024-09-20T18:30:24.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 92799 12.70 MB/sec execute 75 sec latency 52.474 ms 2024-09-20T18:30:24.206 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:24 smithi012 ceph-mon[23745]: pgmap v340: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 827 op/s 2024-09-20T18:30:24.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:24 smithi174 ceph-mon[25682]: pgmap v340: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 827 op/s 2024-09-20T18:30:25.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 92999 12.58 MB/sec execute 76 sec latency 78.244 ms 2024-09-20T18:30:26.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 93371 12.51 MB/sec execute 77 sec latency 57.192 ms 2024-09-20T18:30:26.232 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:26 smithi012 ceph-mon[23745]: pgmap v341: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.7 MiB/s wr, 587 op/s 2024-09-20T18:30:26.518 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:26 smithi174 ceph-mon[25682]: pgmap v341: 97 pgs: 97 active+clean; 335 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.7 MiB/s wr, 587 op/s 2024-09-20T18:30:27.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 93735 12.38 MB/sec execute 78 sec latency 56.504 ms 2024-09-20T18:30:28.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 94086 12.39 MB/sec execute 79 sec latency 47.913 ms 2024-09-20T18:30:28.304 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:28 smithi012 ceph-mon[23745]: pgmap v342: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 571 op/s 2024-09-20T18:30:28.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:28 smithi174 ceph-mon[25682]: pgmap v342: 97 pgs: 97 active+clean; 334 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.4 MiB/s wr, 571 op/s 2024-09-20T18:30:29.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 94566 12.40 MB/sec execute 80 sec latency 43.327 ms 2024-09-20T18:30:30.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 94983 12.35 MB/sec execute 81 sec latency 57.230 ms 2024-09-20T18:30:30.326 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:30 smithi012 ceph-mon[23745]: pgmap v343: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 704 op/s 2024-09-20T18:30:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:30 smithi174 ceph-mon[25682]: pgmap v343: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 704 op/s 2024-09-20T18:30:31.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 95277 12.33 MB/sec execute 82 sec latency 75.006 ms 2024-09-20T18:30:32.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 95596 12.36 MB/sec execute 83 sec latency 59.031 ms 2024-09-20T18:30:32.144 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:31 smithi012 ceph-mon[23745]: pgmap v344: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 502 op/s 2024-09-20T18:30:32.268 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:31 smithi174 ceph-mon[25682]: pgmap v344: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 502 op/s 2024-09-20T18:30:33.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 95946 12.33 MB/sec execute 84 sec latency 38.009 ms 2024-09-20T18:30:34.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 96283 12.31 MB/sec execute 85 sec latency 37.178 ms 2024-09-20T18:30:34.177 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:34 smithi012 ceph-mon[23745]: pgmap v345: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-20T18:30:34.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:34 smithi174 ceph-mon[25682]: pgmap v345: 97 pgs: 97 active+clean; 328 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.4 MiB/s wr, 529 op/s 2024-09-20T18:30:35.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 96503 12.22 MB/sec execute 86 sec latency 64.991 ms 2024-09-20T18:30:36.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 96790 12.16 MB/sec execute 87 sec latency 82.707 ms 2024-09-20T18:30:36.199 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:36 smithi012 ceph-mon[23745]: pgmap v346: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 453 op/s 2024-09-20T18:30:36.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:36 smithi174 ceph-mon[25682]: pgmap v346: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.8 MiB/s wr, 453 op/s 2024-09-20T18:30:37.068 INFO:teuthology.orchestra.run.smithi012.stdout: 5 97146 12.05 MB/sec execute 88 sec latency 68.792 ms 2024-09-20T18:30:38.068 INFO:teuthology.orchestra.run.smithi012.stdout: 5 97448 11.98 MB/sec execute 89 sec latency 42.023 ms 2024-09-20T18:30:38.265 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:38 smithi012 ceph-mon[23745]: pgmap v347: 97 pgs: 97 active+clean; 341 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-09-20T18:30:38.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:38 smithi174 ceph-mon[25682]: pgmap v347: 97 pgs: 97 active+clean; 341 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.1 MiB/s wr, 517 op/s 2024-09-20T18:30:39.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 97766 11.98 MB/sec execute 90 sec latency 42.452 ms 2024-09-20T18:30:40.067 INFO:teuthology.orchestra.run.smithi012.stdout: 5 98188 11.97 MB/sec execute 91 sec latency 43.598 ms 2024-09-20T18:30:40.298 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:40 smithi012 ceph-mon[23745]: pgmap v348: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 591 op/s 2024-09-20T18:30:40.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:40 smithi174 ceph-mon[25682]: pgmap v348: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 591 op/s 2024-09-20T18:30:41.068 INFO:teuthology.orchestra.run.smithi012.stdout: 5 98578 11.91 MB/sec execute 92 sec latency 82.656 ms 2024-09-20T18:30:42.068 INFO:teuthology.orchestra.run.smithi012.stdout: 5 98810 11.89 MB/sec execute 93 sec latency 56.447 ms 2024-09-20T18:30:42.090 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:41 smithi012 ceph-mon[23745]: pgmap v349: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-09-20T18:30:42.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:41 smithi174 ceph-mon[25682]: pgmap v349: 97 pgs: 97 active+clean; 329 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 439 op/s 2024-09-20T18:30:43.068 INFO:teuthology.orchestra.run.smithi012.stdout: 5 99029 11.89 MB/sec execute 94 sec latency 59.522 ms 2024-09-20T18:30:44.068 INFO:teuthology.orchestra.run.smithi012.stdout: 5 99292 11.85 MB/sec execute 95 sec latency 68.595 ms 2024-09-20T18:30:44.145 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:44 smithi012 ceph-mon[23745]: pgmap v350: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 420 op/s 2024-09-20T18:30:44.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:44 smithi174 ceph-mon[25682]: pgmap v350: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 420 op/s 2024-09-20T18:30:45.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 99614 11.83 MB/sec execute 96 sec latency 56.950 ms 2024-09-20T18:30:46.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 99831 11.78 MB/sec execute 97 sec latency 126.545 ms 2024-09-20T18:30:46.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:46 smithi012 ceph-mon[23745]: pgmap v351: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 461 op/s 2024-09-20T18:30:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:46 smithi174 ceph-mon[25682]: pgmap v351: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.8 MiB/s wr, 461 op/s 2024-09-20T18:30:47.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 99972 11.69 MB/sec execute 98 sec latency 80.746 ms 2024-09-20T18:30:48.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 100261 11.65 MB/sec execute 99 sec latency 64.592 ms 2024-09-20T18:30:48.195 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:48 smithi012 ceph-mon[23745]: pgmap v352: 97 pgs: 97 active+clean; 337 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-20T18:30:48.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:48 smithi174 ceph-mon[25682]: pgmap v352: 97 pgs: 97 active+clean; 337 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-20T18:30:49.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 100587 11.55 MB/sec execute 100 sec latency 68.001 ms 2024-09-20T18:30:50.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 100926 11.50 MB/sec execute 101 sec latency 77.574 ms 2024-09-20T18:30:50.234 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:50 smithi012 ceph-mon[23745]: pgmap v353: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.4 MiB/s wr, 415 op/s 2024-09-20T18:30:50.375 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:30:50.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:50 smithi174 ceph-mon[25682]: pgmap v353: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.4 MiB/s wr, 415 op/s 2024-09-20T18:30:51.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 101248 11.48 MB/sec execute 102 sec latency 96.454 ms 2024-09-20T18:30:51.266 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:51 smithi012 ceph-mon[23745]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (8m) 3m ago 10m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (10m) 3m ago 10m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (9m) 3m ago 9m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (8m) 3m ago 10m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (4m) 3m ago 4m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (4m) 3m ago 4m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (11m) 3m ago 11m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (9m) 3m ago 9m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:30:51.319 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (11m) 3m ago 11m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (9m) 3m ago 9m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (3m) 3m ago 3m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (3m) 3m ago 3m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (10m) 3m ago 9m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (9m) 3m ago 9m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (8m) 3m ago 8m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (8m) 3m ago 8m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (7m) 3m ago 7m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:30:51.320 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (7m) 3m ago 7m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:30:51.321 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (6m) 3m ago 6m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:30:51.321 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (6m) 3m ago 6m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:30:51.321 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (6m) 3m ago 6m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:30:51.321 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (5m) 3m ago 5m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:30:51.321 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (9m) 3m ago 9m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:30:51.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:51 smithi174 ceph-mon[25682]: from='client.14748 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:51.864 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:30:51.864 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:30:51.864 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:30:51.864 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:51.865 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:30:51.865 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:30:51.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:51.865 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:30:51.865 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:30:51.865 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:51.865 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:30:51.866 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:30:51.866 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:30:51.866 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:30:51.866 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:30:51.866 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:30:51.866 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:30:52.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 101725 11.51 MB/sec execute 103 sec latency 51.432 ms 2024-09-20T18:30:52.152 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:52 smithi012 ceph-mon[23745]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:52.152 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:52 smithi012 ceph-mon[23745]: pgmap v354: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 371 op/s 2024-09-20T18:30:52.152 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:52 smithi012 ceph-mon[23745]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:52.152 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:52 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/4065740216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:30:52.338 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:30:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:30:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:30:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:30:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:30:52.338 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:30:52.339 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:30:52.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:52 smithi174 ceph-mon[25682]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:52.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:52 smithi174 ceph-mon[25682]: pgmap v354: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.1 MiB/s wr, 371 op/s 2024-09-20T18:30:52.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:52 smithi174 ceph-mon[25682]: from='client.14756 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:52.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:52 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/4065740216' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:30:52.900 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:30:53.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 102125 11.49 MB/sec execute 104 sec latency 43.238 ms 2024-09-20T18:30:53.277 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:53 smithi012 ceph-mon[23745]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:53.277 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:53 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/531133981' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:30:53.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:53 smithi174 ceph-mon[25682]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:30:53.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:53 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/531133981' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:30:54.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 102439 11.49 MB/sec execute 105 sec latency 63.252 ms 2024-09-20T18:30:54.305 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:54 smithi012 ceph-mon[23745]: pgmap v355: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.0 MiB/s wr, 463 op/s 2024-09-20T18:30:54.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:54 smithi174 ceph-mon[25682]: pgmap v355: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.0 MiB/s wr, 463 op/s 2024-09-20T18:30:55.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 102771 11.50 MB/sec execute 106 sec latency 129.376 ms 2024-09-20T18:30:56.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 103092 11.47 MB/sec execute 107 sec latency 92.111 ms 2024-09-20T18:30:56.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:56 smithi012 ceph-mon[23745]: pgmap v356: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 534 op/s 2024-09-20T18:30:56.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:56 smithi174 ceph-mon[25682]: pgmap v356: 97 pgs: 97 active+clean; 331 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 534 op/s 2024-09-20T18:30:57.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 103373 11.45 MB/sec execute 108 sec latency 71.636 ms 2024-09-20T18:30:58.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 103578 11.40 MB/sec execute 109 sec latency 62.058 ms 2024-09-20T18:30:58.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:58 smithi012 ceph-mon[23745]: pgmap v357: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 468 op/s 2024-09-20T18:30:58.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:58 smithi174 ceph-mon[25682]: pgmap v357: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 468 op/s 2024-09-20T18:30:59.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 103814 11.35 MB/sec execute 110 sec latency 124.793 ms 2024-09-20T18:30:59.207 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:30:59.207 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:30:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:30:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:30:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:30:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:31:00.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 104284 11.28 MB/sec execute 111 sec latency 64.052 ms 2024-09-20T18:31:00.212 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:00 smithi012 ceph-mon[23745]: pgmap v358: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 386 op/s 2024-09-20T18:31:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:00 smithi174 ceph-mon[25682]: pgmap v358: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.5 MiB/s wr, 386 op/s 2024-09-20T18:31:01.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 104543 11.23 MB/sec execute 112 sec latency 92.645 ms 2024-09-20T18:31:01.992 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:01 smithi012 ceph-mon[23745]: pgmap v359: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 475 op/s 2024-09-20T18:31:02.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 104994 11.31 MB/sec execute 113 sec latency 39.844 ms 2024-09-20T18:31:02.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:01 smithi174 ceph-mon[25682]: pgmap v359: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 475 op/s 2024-09-20T18:31:03.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 105493 11.27 MB/sec execute 114 sec latency 42.628 ms 2024-09-20T18:31:04.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 105843 11.27 MB/sec execute 115 sec latency 69.492 ms 2024-09-20T18:31:04.278 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:04 smithi012 ceph-mon[23745]: pgmap v360: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 581 op/s 2024-09-20T18:31:04.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:04 smithi174 ceph-mon[25682]: pgmap v360: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 581 op/s 2024-09-20T18:31:05.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 106176 11.30 MB/sec execute 116 sec latency 104.959 ms 2024-09-20T18:31:06.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 106445 11.25 MB/sec execute 117 sec latency 90.753 ms 2024-09-20T18:31:06.325 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:06 smithi012 ceph-mon[23745]: pgmap v361: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 480 op/s 2024-09-20T18:31:06.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:06 smithi174 ceph-mon[25682]: pgmap v361: 97 pgs: 97 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 480 op/s 2024-09-20T18:31:07.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 106884 11.28 MB/sec execute 118 sec latency 42.104 ms 2024-09-20T18:31:08.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 107169 11.27 MB/sec execute 119 sec latency 43.032 ms 2024-09-20T18:31:08.146 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:08 smithi012 ceph-mon[23745]: pgmap v362: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 370 op/s 2024-09-20T18:31:08.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:08 smithi174 ceph-mon[25682]: pgmap v362: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 370 op/s 2024-09-20T18:31:09.071 INFO:teuthology.orchestra.run.smithi012.stdout: 5 107558 11.21 MB/sec execute 120 sec latency 50.494 ms 2024-09-20T18:31:10.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 107879 11.15 MB/sec execute 121 sec latency 66.521 ms 2024-09-20T18:31:10.212 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:10 smithi012 ceph-mon[23745]: pgmap v363: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 513 op/s 2024-09-20T18:31:10.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:10 smithi174 ceph-mon[25682]: pgmap v363: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 513 op/s 2024-09-20T18:31:11.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 108167 11.13 MB/sec execute 122 sec latency 64.594 ms 2024-09-20T18:31:11.995 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:11 smithi012 ceph-mon[23745]: pgmap v364: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 491 op/s 2024-09-20T18:31:12.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 108542 11.16 MB/sec execute 123 sec latency 55.544 ms 2024-09-20T18:31:12.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:11 smithi174 ceph-mon[25682]: pgmap v364: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 491 op/s 2024-09-20T18:31:13.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 108926 11.11 MB/sec execute 124 sec latency 58.816 ms 2024-09-20T18:31:14.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 109260 11.09 MB/sec execute 125 sec latency 71.326 ms 2024-09-20T18:31:14.292 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:14 smithi012 ceph-mon[23745]: pgmap v365: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-09-20T18:31:14.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:14 smithi174 ceph-mon[25682]: pgmap v365: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-09-20T18:31:15.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 109457 11.06 MB/sec execute 126 sec latency 50.675 ms 2024-09-20T18:31:16.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 109739 11.08 MB/sec execute 127 sec latency 148.760 ms 2024-09-20T18:31:16.391 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:16 smithi012 ceph-mon[23745]: pgmap v366: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 421 op/s 2024-09-20T18:31:16.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:16 smithi174 ceph-mon[25682]: pgmap v366: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 421 op/s 2024-09-20T18:31:17.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 110101 11.05 MB/sec execute 128 sec latency 77.892 ms 2024-09-20T18:31:18.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 110428 11.07 MB/sec execute 129 sec latency 75.196 ms 2024-09-20T18:31:18.196 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:18 smithi012 ceph-mon[23745]: pgmap v367: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 533 op/s 2024-09-20T18:31:18.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:18 smithi174 ceph-mon[25682]: pgmap v367: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 533 op/s 2024-09-20T18:31:19.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 110669 11.05 MB/sec execute 130 sec latency 61.317 ms 2024-09-20T18:31:20.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 111002 11.00 MB/sec execute 131 sec latency 44.494 ms 2024-09-20T18:31:20.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:20 smithi012 ceph-mon[23745]: pgmap v368: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 529 op/s 2024-09-20T18:31:20.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:20 smithi174 ceph-mon[25682]: pgmap v368: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 529 op/s 2024-09-20T18:31:21.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 111362 10.94 MB/sec execute 132 sec latency 64.289 ms 2024-09-20T18:31:22.031 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:21 smithi012 ceph-mon[23745]: pgmap v369: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 472 op/s 2024-09-20T18:31:22.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 111592 10.90 MB/sec execute 133 sec latency 70.101 ms 2024-09-20T18:31:22.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:21 smithi174 ceph-mon[25682]: pgmap v369: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 472 op/s 2024-09-20T18:31:23.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 111904 10.91 MB/sec execute 134 sec latency 53.162 ms 2024-09-20T18:31:23.317 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:31:24.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 112242 10.90 MB/sec execute 135 sec latency 73.296 ms 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (9m) 4m ago 10m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (10m) 4m ago 10m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (10m) 4m ago 10m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (9m) 4m ago 10m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (4m) 4m ago 4m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (4m) 4m ago 4m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (12m) 4m ago 12m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (10m) 4m ago 10m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (12m) 4m ago 12m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (10m) 4m ago 10m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (4m) 4m ago 4m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:31:24.158 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (4m) 4m ago 4m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (10m) 4m ago 10m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (9m) 4m ago 9m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (9m) 4m ago 9m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (8m) 4m ago 8m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (8m) 4m ago 8m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (7m) 4m ago 7m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (7m) 4m ago 7m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (7m) 4m ago 7m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (6m) 4m ago 6m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (6m) 4m ago 6m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:31:24.159 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (9m) 4m ago 9m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:31:24.325 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:24 smithi012 ceph-mon[23745]: pgmap v370: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-09-20T18:31:24.325 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:24 smithi012 ceph-mon[23745]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:24.325 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:24 smithi012 ceph-mon[23745]: from='client.24503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:24.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:24 smithi174 ceph-mon[25682]: pgmap v370: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.1 MiB/s wr, 511 op/s 2024-09-20T18:31:24.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:24 smithi174 ceph-mon[25682]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:24.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:24 smithi174 ceph-mon[25682]: from='client.24503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:24.645 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:31:24.645 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:31:24.645 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:31:24.646 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:31:24.647 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:31:25.072 INFO:teuthology.orchestra.run.smithi012.stdout: 5 112671 10.88 MB/sec execute 136 sec latency 67.282 ms 2024-09-20T18:31:25.084 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:31:25.084 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:31:25.084 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:31:25.084 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:31:25.084 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:31:25.084 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:31:25.085 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:31:25.335 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:25 smithi012 ceph-mon[23745]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:25.335 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:25 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2160508158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:31:25.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:25 smithi174 ceph-mon[25682]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:25.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:25 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2160508158' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:31:25.622 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:31:26.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 112968 10.87 MB/sec execute 137 sec latency 43.177 ms 2024-09-20T18:31:26.386 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:26 smithi012 ceph-mon[23745]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:26.387 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:26 smithi012 ceph-mon[23745]: pgmap v371: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 397 op/s 2024-09-20T18:31:26.387 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:26 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2134256484' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:31:26.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:26 smithi174 ceph-mon[25682]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:26.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:26 smithi174 ceph-mon[25682]: pgmap v371: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.1 MiB/s wr, 397 op/s 2024-09-20T18:31:26.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:26 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2134256484' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:31:27.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 113259 10.87 MB/sec execute 138 sec latency 69.428 ms 2024-09-20T18:31:28.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 113646 10.88 MB/sec execute 139 sec latency 61.899 ms 2024-09-20T18:31:28.447 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:28 smithi012 ceph-mon[23745]: pgmap v372: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.1 MiB/s wr, 520 op/s 2024-09-20T18:31:28.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:28 smithi174 ceph-mon[25682]: pgmap v372: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.1 MiB/s wr, 520 op/s 2024-09-20T18:31:29.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 113971 10.87 MB/sec execute 140 sec latency 78.574 ms 2024-09-20T18:31:30.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 114292 10.87 MB/sec execute 141 sec latency 52.622 ms 2024-09-20T18:31:30.220 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:30 smithi012 ceph-mon[23745]: pgmap v373: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 428 op/s 2024-09-20T18:31:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:30 smithi174 ceph-mon[25682]: pgmap v373: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 428 op/s 2024-09-20T18:31:31.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 114660 10.82 MB/sec execute 142 sec latency 44.119 ms 2024-09-20T18:31:31.999 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:31 smithi012 ceph-mon[23745]: pgmap v374: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 509 op/s 2024-09-20T18:31:32.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 114971 10.77 MB/sec execute 143 sec latency 70.987 ms 2024-09-20T18:31:32.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:31 smithi174 ceph-mon[25682]: pgmap v374: 97 pgs: 97 active+clean; 342 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.8 MiB/s wr, 509 op/s 2024-09-20T18:31:33.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 115320 10.78 MB/sec execute 144 sec latency 69.127 ms 2024-09-20T18:31:34.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 115638 10.75 MB/sec execute 145 sec latency 46.530 ms 2024-09-20T18:31:34.311 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:34 smithi012 ceph-mon[23745]: pgmap v375: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 452 op/s 2024-09-20T18:31:34.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:34 smithi174 ceph-mon[25682]: pgmap v375: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.0 MiB/s wr, 452 op/s 2024-09-20T18:31:35.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 116052 10.77 MB/sec execute 146 sec latency 47.484 ms 2024-09-20T18:31:36.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 116387 10.74 MB/sec execute 147 sec latency 42.188 ms 2024-09-20T18:31:36.333 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:36 smithi012 ceph-mon[23745]: pgmap v376: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 471 op/s 2024-09-20T18:31:36.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:36 smithi174 ceph-mon[25682]: pgmap v376: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 471 op/s 2024-09-20T18:31:37.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 116694 10.76 MB/sec execute 148 sec latency 71.298 ms 2024-09-20T18:31:38.073 INFO:teuthology.orchestra.run.smithi012.stdout: 5 117028 10.75 MB/sec execute 149 sec latency 69.869 ms 2024-09-20T18:31:38.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:38 smithi012 ceph-mon[23745]: pgmap v377: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.0 MiB/s wr, 532 op/s 2024-09-20T18:31:38.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:38 smithi174 ceph-mon[25682]: pgmap v377: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.0 MiB/s wr, 532 op/s 2024-09-20T18:31:39.077 INFO:teuthology.orchestra.run.smithi012.stdout: 5 117322 10.75 MB/sec execute 150 sec latency 68.529 ms 2024-09-20T18:31:40.077 INFO:teuthology.orchestra.run.smithi012.stdout: 5 117523 10.71 MB/sec execute 151 sec latency 72.037 ms 2024-09-20T18:31:40.244 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:40 smithi012 ceph-mon[23745]: pgmap v378: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-09-20T18:31:40.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:40 smithi174 ceph-mon[25682]: pgmap v378: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.6 MiB/s wr, 454 op/s 2024-09-20T18:31:41.077 INFO:teuthology.orchestra.run.smithi012.stdout: 5 117789 10.70 MB/sec execute 152 sec latency 52.439 ms 2024-09-20T18:31:42.018 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:41 smithi012 ceph-mon[23745]: pgmap v379: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-20T18:31:42.077 INFO:teuthology.orchestra.run.smithi012.stdout: 5 118141 10.67 MB/sec execute 153 sec latency 43.703 ms 2024-09-20T18:31:42.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:41 smithi174 ceph-mon[25682]: pgmap v379: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-20T18:31:43.077 INFO:teuthology.orchestra.run.smithi012.stdout: 5 118482 10.62 MB/sec execute 154 sec latency 44.122 ms 2024-09-20T18:31:44.083 INFO:teuthology.orchestra.run.smithi012.stdout: 5 118884 10.63 MB/sec execute 155 sec latency 86.853 ms 2024-09-20T18:31:44.335 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:44 smithi012 ceph-mon[23745]: pgmap v380: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.9 MiB/s wr, 436 op/s 2024-09-20T18:31:44.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:44 smithi174 ceph-mon[25682]: pgmap v380: 97 pgs: 97 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.9 MiB/s wr, 436 op/s 2024-09-20T18:31:45.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 119359 10.64 MB/sec execute 156 sec latency 42.889 ms 2024-09-20T18:31:46.083 INFO:teuthology.orchestra.run.smithi012.stdout: 5 119766 10.64 MB/sec execute 157 sec latency 50.243 ms 2024-09-20T18:31:46.380 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:46 smithi012 ceph-mon[23745]: pgmap v381: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 467 op/s 2024-09-20T18:31:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:46 smithi174 ceph-mon[25682]: pgmap v381: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 467 op/s 2024-09-20T18:31:47.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 120010 10.62 MB/sec execute 158 sec latency 62.367 ms 2024-09-20T18:31:48.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 120276 10.62 MB/sec execute 159 sec latency 51.603 ms 2024-09-20T18:31:48.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:48 smithi012 ceph-mon[23745]: pgmap v382: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.0 MiB/s wr, 540 op/s 2024-09-20T18:31:48.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:48 smithi174 ceph-mon[25682]: pgmap v382: 97 pgs: 97 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.0 MiB/s wr, 540 op/s 2024-09-20T18:31:49.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 120521 10.60 MB/sec execute 160 sec latency 92.189 ms 2024-09-20T18:31:50.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 120783 10.59 MB/sec execute 161 sec latency 49.516 ms 2024-09-20T18:31:50.224 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:50 smithi012 ceph-mon[23745]: pgmap v383: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 445 op/s 2024-09-20T18:31:50.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:50 smithi174 ceph-mon[25682]: pgmap v383: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 445 op/s 2024-09-20T18:31:51.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 121114 10.59 MB/sec execute 162 sec latency 66.665 ms 2024-09-20T18:31:52.017 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:51 smithi012 ceph-mon[23745]: pgmap v384: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 405 op/s 2024-09-20T18:31:52.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 121435 10.57 MB/sec execute 163 sec latency 41.148 ms 2024-09-20T18:31:52.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:51 smithi174 ceph-mon[25682]: pgmap v384: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 405 op/s 2024-09-20T18:31:53.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 121768 10.54 MB/sec execute 164 sec latency 79.074 ms 2024-09-20T18:31:54.084 INFO:teuthology.orchestra.run.smithi012.stdout: 5 122064 10.50 MB/sec execute 165 sec latency 53.413 ms 2024-09-20T18:31:54.262 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:54 smithi012 ceph-mon[23745]: pgmap v385: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 540 op/s 2024-09-20T18:31:54.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:54 smithi174 ceph-mon[25682]: pgmap v385: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 540 op/s 2024-09-20T18:31:55.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 122417 10.50 MB/sec execute 166 sec latency 49.629 ms 2024-09-20T18:31:56.081 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:31:56.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 122692 10.48 MB/sec execute 167 sec latency 62.733 ms 2024-09-20T18:31:56.307 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:56 smithi012 ceph-mon[23745]: pgmap v386: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 444 op/s 2024-09-20T18:31:56.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:56 smithi174 ceph-mon[25682]: pgmap v386: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.5 MiB/s wr, 444 op/s 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (9m) 4m ago 11m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (11m) 4m ago 11m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (10m) 4m ago 10m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (9m) 4m ago 11m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (5m) 4m ago 5m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (5m) 4m ago 5m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (12m) 4m ago 12m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (10m) 4m ago 10m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (12m) 4m ago 12m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:31:56.912 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (10m) 4m ago 10m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (5m) 4m ago 5m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (4m) 4m ago 4m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (11m) 4m ago 11m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (10m) 4m ago 10m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (9m) 4m ago 9m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (9m) 4m ago 9m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (8m) 4m ago 8m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (8m) 4m ago 8m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (8m) 4m ago 8m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (7m) 4m ago 7m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (7m) 4m ago 7m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (6m) 4m ago 6m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:31:56.913 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (10m) 4m ago 10m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:31:57.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 123151 10.50 MB/sec execute 168 sec latency 102.418 ms 2024-09-20T18:31:57.342 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:57 smithi012 ceph-mon[23745]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:57.342 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:57 smithi012 ceph-mon[23745]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:57.411 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:31:57.412 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:31:57.412 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:31:57.412 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:31:57.412 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:31:57.412 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:31:57.412 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:31:57.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:57 smithi174 ceph-mon[25682]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:57.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:57 smithi174 ceph-mon[25682]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:57.800 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:31:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:31:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:31:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:31:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:31:57.801 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:31:57.801 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:31:58.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 123454 10.47 MB/sec execute 169 sec latency 42.423 ms 2024-09-20T18:31:58.276 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:31:58.356 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:58 smithi012 ceph-mon[23745]: from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:58.356 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:58 smithi012 ceph-mon[23745]: pgmap v387: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 444 op/s 2024-09-20T18:31:58.356 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:58 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1990460201' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:31:58.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:58 smithi174 ceph-mon[25682]: from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:58.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:58 smithi174 ceph-mon[25682]: pgmap v387: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 444 op/s 2024-09-20T18:31:58.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:58 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1990460201' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:31:59.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 123812 10.49 MB/sec execute 170 sec latency 64.747 ms 2024-09-20T18:31:59.383 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:59 smithi012 ceph-mon[23745]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:59.383 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:59 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2548359729' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:31:59.383 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:31:59.383 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:31:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:31:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:59 smithi174 ceph-mon[25682]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:31:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:59 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2548359729' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:31:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:31:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:31:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:32:00.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 124106 10.48 MB/sec execute 171 sec latency 54.542 ms 2024-09-20T18:32:00.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:00 smithi012 ceph-mon[23745]: pgmap v388: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 433 op/s 2024-09-20T18:32:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:00 smithi174 ceph-mon[25682]: pgmap v388: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.8 MiB/s wr, 433 op/s 2024-09-20T18:32:01.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 124426 10.49 MB/sec execute 172 sec latency 54.058 ms 2024-09-20T18:32:02.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 124648 10.46 MB/sec execute 173 sec latency 135.134 ms 2024-09-20T18:32:02.184 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:01 smithi012 ceph-mon[23745]: pgmap v389: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 411 op/s 2024-09-20T18:32:02.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:01 smithi174 ceph-mon[25682]: pgmap v389: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 411 op/s 2024-09-20T18:32:03.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 124894 10.44 MB/sec execute 174 sec latency 65.431 ms 2024-09-20T18:32:04.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 125155 10.40 MB/sec execute 175 sec latency 66.963 ms 2024-09-20T18:32:04.264 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:04 smithi012 ceph-mon[23745]: pgmap v390: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 585 op/s 2024-09-20T18:32:04.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:04 smithi174 ceph-mon[25682]: pgmap v390: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 585 op/s 2024-09-20T18:32:05.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 125388 10.37 MB/sec execute 176 sec latency 112.790 ms 2024-09-20T18:32:06.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 125699 10.33 MB/sec execute 177 sec latency 76.376 ms 2024-09-20T18:32:06.300 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:06 smithi012 ceph-mon[23745]: pgmap v391: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-09-20T18:32:06.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:06 smithi174 ceph-mon[25682]: pgmap v391: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 436 op/s 2024-09-20T18:32:07.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 126013 10.34 MB/sec execute 178 sec latency 60.961 ms 2024-09-20T18:32:08.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 126413 10.34 MB/sec execute 179 sec latency 60.791 ms 2024-09-20T18:32:08.337 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:08 smithi012 ceph-mon[23745]: pgmap v392: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 508 op/s 2024-09-20T18:32:08.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:08 smithi174 ceph-mon[25682]: pgmap v392: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.8 MiB/s wr, 508 op/s 2024-09-20T18:32:09.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 126845 10.35 MB/sec execute 180 sec latency 83.323 ms 2024-09-20T18:32:10.087 INFO:teuthology.orchestra.run.smithi012.stdout: 5 127216 10.35 MB/sec execute 181 sec latency 68.747 ms 2024-09-20T18:32:10.386 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:10 smithi012 ceph-mon[23745]: pgmap v393: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.6 MiB/s wr, 473 op/s 2024-09-20T18:32:10.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:10 smithi174 ceph-mon[25682]: pgmap v393: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.6 MiB/s wr, 473 op/s 2024-09-20T18:32:11.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 127537 10.35 MB/sec execute 182 sec latency 40.350 ms 2024-09-20T18:32:12.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 127858 10.36 MB/sec execute 183 sec latency 62.146 ms 2024-09-20T18:32:12.191 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:11 smithi012 ceph-mon[23745]: pgmap v394: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 361 op/s 2024-09-20T18:32:12.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:11 smithi174 ceph-mon[25682]: pgmap v394: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 361 op/s 2024-09-20T18:32:13.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 128171 10.35 MB/sec execute 184 sec latency 53.673 ms 2024-09-20T18:32:14.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 128438 10.34 MB/sec execute 185 sec latency 53.213 ms 2024-09-20T18:32:14.249 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:14 smithi012 ceph-mon[23745]: pgmap v395: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 553 op/s 2024-09-20T18:32:14.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:14 smithi174 ceph-mon[25682]: pgmap v395: 97 pgs: 97 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 553 op/s 2024-09-20T18:32:15.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 128750 10.31 MB/sec execute 186 sec latency 69.953 ms 2024-09-20T18:32:16.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 129007 10.27 MB/sec execute 187 sec latency 51.855 ms 2024-09-20T18:32:16.291 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:16 smithi012 ceph-mon[23745]: pgmap v396: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 369 op/s 2024-09-20T18:32:16.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:16 smithi174 ceph-mon[25682]: pgmap v396: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 369 op/s 2024-09-20T18:32:17.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 129323 10.26 MB/sec execute 188 sec latency 67.636 ms 2024-09-20T18:32:18.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 129610 10.25 MB/sec execute 189 sec latency 55.584 ms 2024-09-20T18:32:18.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:18 smithi174 ceph-mon[25682]: pgmap v397: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-20T18:32:18.366 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:18 smithi012 ceph-mon[23745]: pgmap v397: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.6 MiB/s wr, 450 op/s 2024-09-20T18:32:19.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 130149 10.28 MB/sec execute 190 sec latency 36.142 ms 2024-09-20T18:32:20.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 130467 10.26 MB/sec execute 191 sec latency 74.630 ms 2024-09-20T18:32:20.397 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:20 smithi012 ceph-mon[23745]: pgmap v398: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.8 MiB/s wr, 465 op/s 2024-09-20T18:32:20.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:20 smithi174 ceph-mon[25682]: pgmap v398: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.8 MiB/s wr, 465 op/s 2024-09-20T18:32:21.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 130904 10.29 MB/sec execute 192 sec latency 24.621 ms 2024-09-20T18:32:22.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 131211 10.27 MB/sec execute 193 sec latency 63.338 ms 2024-09-20T18:32:22.180 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:21 smithi012 ceph-mon[23745]: pgmap v399: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 436 op/s 2024-09-20T18:32:22.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:21 smithi174 ceph-mon[25682]: pgmap v399: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 436 op/s 2024-09-20T18:32:23.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 131472 10.26 MB/sec execute 194 sec latency 76.810 ms 2024-09-20T18:32:24.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 131712 10.25 MB/sec execute 195 sec latency 65.031 ms 2024-09-20T18:32:24.234 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:24 smithi012 ceph-mon[23745]: pgmap v400: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 609 op/s 2024-09-20T18:32:24.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:24 smithi174 ceph-mon[25682]: pgmap v400: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 609 op/s 2024-09-20T18:32:25.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 131985 10.25 MB/sec execute 196 sec latency 47.924 ms 2024-09-20T18:32:26.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 132454 10.23 MB/sec execute 197 sec latency 33.557 ms 2024-09-20T18:32:26.265 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:26 smithi012 ceph-mon[23745]: pgmap v401: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 409 op/s 2024-09-20T18:32:26.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:26 smithi174 ceph-mon[25682]: pgmap v401: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 409 op/s 2024-09-20T18:32:27.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 132893 10.23 MB/sec execute 198 sec latency 25.118 ms 2024-09-20T18:32:28.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 133304 10.24 MB/sec execute 199 sec latency 53.663 ms 2024-09-20T18:32:28.287 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:28 smithi012 ceph-mon[23745]: pgmap v402: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 569 op/s 2024-09-20T18:32:28.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:28 smithi174 ceph-mon[25682]: pgmap v402: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.3 MiB/s wr, 569 op/s 2024-09-20T18:32:28.764 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:32:29.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 133742 10.25 MB/sec execute 200 sec latency 43.294 ms 2024-09-20T18:32:29.620 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:32:29.620 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (10m) 5m ago 11m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:32:29.621 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (11m) 5m ago 11m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:32:29.621 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (11m) 5m ago 11m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:32:29.621 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (10m) 5m ago 11m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:32:29.621 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (5m) 5m ago 5m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:32:29.621 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (5m) 5m ago 5m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:32:29.621 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (13m) 5m ago 13m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:32:29.621 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (11m) 5m ago 11m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (13m) 5m ago 13m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (11m) 5m ago 11m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (5m) 5m ago 5m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (5m) 5m ago 5m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (11m) 5m ago 11m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (11m) 5m ago 11m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (10m) 5m ago 10m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (9m) 5m ago 9m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (9m) 5m ago 9m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (8m) 5m ago 8m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (8m) 5m ago 8m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (8m) 5m ago 8m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (7m) 5m ago 7m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (7m) 5m ago 7m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:32:29.622 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (10m) 5m ago 10m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:32:30.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 134018 10.23 MB/sec execute 201 sec latency 80.732 ms 2024-09-20T18:32:30.108 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:32:30.108 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:32:30.108 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:32:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:32:30.110 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:32:30.220 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:30 smithi012 ceph-mon[23745]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:30.220 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:30 smithi012 ceph-mon[23745]: pgmap v403: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 519 op/s 2024-09-20T18:32:30.220 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:30 smithi012 ceph-mon[23745]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:30.220 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:30 smithi012 ceph-mon[23745]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:30.221 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:30 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2254351114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:32:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:30 smithi174 ceph-mon[25682]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:30 smithi174 ceph-mon[25682]: pgmap v403: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 519 op/s 2024-09-20T18:32:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:30 smithi174 ceph-mon[25682]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:30 smithi174 ceph-mon[25682]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:30 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2254351114' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:32:30.533 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:32:30.534 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:32:30.534 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:32:30.534 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:32:30.534 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:32:30.534 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:32:30.534 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:32:31.040 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:32:31.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 134325 10.23 MB/sec execute 202 sec latency 50.825 ms 2024-09-20T18:32:31.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:31 smithi012 ceph-mon[23745]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:31.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:31 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/4210499302' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:32:31.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:31 smithi174 ceph-mon[25682]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:32:31.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:31 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/4210499302' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:32:32.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 134626 10.22 MB/sec execute 203 sec latency 41.460 ms 2024-09-20T18:32:32.268 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:32 smithi012 ceph-mon[23745]: pgmap v404: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 451 op/s 2024-09-20T18:32:32.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:32 smithi174 ceph-mon[25682]: pgmap v404: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 451 op/s 2024-09-20T18:32:33.088 INFO:teuthology.orchestra.run.smithi012.stdout: 5 134950 10.23 MB/sec execute 204 sec latency 37.433 ms 2024-09-20T18:32:34.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 135274 10.22 MB/sec execute 205 sec latency 41.847 ms 2024-09-20T18:32:34.321 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:34 smithi012 ceph-mon[23745]: pgmap v405: 97 pgs: 97 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.1 MiB/s wr, 590 op/s 2024-09-20T18:32:34.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:34 smithi174 ceph-mon[25682]: pgmap v405: 97 pgs: 97 active+clean; 350 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.1 MiB/s wr, 590 op/s 2024-09-20T18:32:35.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 135609 10.22 MB/sec execute 206 sec latency 106.394 ms 2024-09-20T18:32:36.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 136001 10.20 MB/sec execute 207 sec latency 50.270 ms 2024-09-20T18:32:36.352 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:36 smithi012 ceph-mon[23745]: pgmap v406: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-09-20T18:32:36.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:36 smithi174 ceph-mon[25682]: pgmap v406: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-09-20T18:32:37.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 136421 10.19 MB/sec execute 208 sec latency 31.794 ms 2024-09-20T18:32:38.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 136692 10.18 MB/sec execute 209 sec latency 78.172 ms 2024-09-20T18:32:38.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:38 smithi012 ceph-mon[23745]: pgmap v407: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.2 MiB/s wr, 502 op/s 2024-09-20T18:32:38.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:38 smithi174 ceph-mon[25682]: pgmap v407: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.2 MiB/s wr, 502 op/s 2024-09-20T18:32:39.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 137177 10.21 MB/sec execute 210 sec latency 47.006 ms 2024-09-20T18:32:40.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 137532 10.19 MB/sec execute 211 sec latency 59.861 ms 2024-09-20T18:32:40.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:40 smithi012 ceph-mon[23745]: pgmap v408: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.1 MiB/s wr, 468 op/s 2024-09-20T18:32:40.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:40 smithi174 ceph-mon[25682]: pgmap v408: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.1 MiB/s wr, 468 op/s 2024-09-20T18:32:41.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 137894 10.20 MB/sec execute 212 sec latency 77.243 ms 2024-09-20T18:32:42.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 138311 10.21 MB/sec execute 213 sec latency 64.117 ms 2024-09-20T18:32:42.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:41 smithi012 ceph-mon[23745]: pgmap v409: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 453 op/s 2024-09-20T18:32:42.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:41 smithi174 ceph-mon[25682]: pgmap v409: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 453 op/s 2024-09-20T18:32:43.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 138665 10.22 MB/sec execute 214 sec latency 45.337 ms 2024-09-20T18:32:44.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 138990 10.23 MB/sec execute 215 sec latency 54.419 ms 2024-09-20T18:32:44.271 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:44 smithi012 ceph-mon[23745]: pgmap v410: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-09-20T18:32:44.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:44 smithi174 ceph-mon[25682]: pgmap v410: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-09-20T18:32:45.089 INFO:teuthology.orchestra.run.smithi012.stdout: 5 139285 10.19 MB/sec execute 216 sec latency 61.713 ms 2024-09-20T18:32:46.093 INFO:teuthology.orchestra.run.smithi012.stdout: 5 139580 10.17 MB/sec execute 217 sec latency 67.052 ms 2024-09-20T18:32:46.323 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:46 smithi012 ceph-mon[23745]: pgmap v411: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 508 op/s 2024-09-20T18:32:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:46 smithi174 ceph-mon[25682]: pgmap v411: 97 pgs: 97 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 508 op/s 2024-09-20T18:32:47.093 INFO:teuthology.orchestra.run.smithi012.stdout: 5 139882 10.14 MB/sec execute 218 sec latency 109.079 ms 2024-09-20T18:32:48.093 INFO:teuthology.orchestra.run.smithi012.stdout: 5 140207 10.15 MB/sec execute 219 sec latency 56.317 ms 2024-09-20T18:32:48.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:48 smithi174 ceph-mon[25682]: pgmap v412: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.4 MiB/s wr, 525 op/s 2024-09-20T18:32:48.360 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:48 smithi012 ceph-mon[23745]: pgmap v412: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.4 MiB/s wr, 525 op/s 2024-09-20T18:32:49.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 140617 10.16 MB/sec execute 220 sec latency 45.450 ms 2024-09-20T18:32:50.093 INFO:teuthology.orchestra.run.smithi012.stdout: 5 141089 10.16 MB/sec execute 221 sec latency 63.051 ms 2024-09-20T18:32:50.397 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:50 smithi012 ceph-mon[23745]: pgmap v413: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 552 op/s 2024-09-20T18:32:50.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:50 smithi174 ceph-mon[25682]: pgmap v413: 97 pgs: 97 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.5 MiB/s wr, 552 op/s 2024-09-20T18:32:51.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 141458 10.17 MB/sec execute 222 sec latency 64.197 ms 2024-09-20T18:32:52.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 141788 10.17 MB/sec execute 223 sec latency 56.170 ms 2024-09-20T18:32:52.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:51 smithi012 ceph-mon[23745]: pgmap v414: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 443 op/s 2024-09-20T18:32:52.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:51 smithi174 ceph-mon[25682]: pgmap v414: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 443 op/s 2024-09-20T18:32:53.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 142162 10.18 MB/sec execute 224 sec latency 40.328 ms 2024-09-20T18:32:54.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 142503 10.19 MB/sec execute 225 sec latency 56.017 ms 2024-09-20T18:32:54.256 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:54 smithi012 ceph-mon[23745]: pgmap v415: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 558 op/s 2024-09-20T18:32:54.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:54 smithi174 ceph-mon[25682]: pgmap v415: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.8 MiB/s wr, 558 op/s 2024-09-20T18:32:55.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 142888 10.18 MB/sec execute 226 sec latency 38.470 ms 2024-09-20T18:32:56.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 143191 10.15 MB/sec execute 227 sec latency 47.467 ms 2024-09-20T18:32:56.308 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:56 smithi012 ceph-mon[23745]: pgmap v416: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.9 MiB/s wr, 558 op/s 2024-09-20T18:32:56.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:56 smithi174 ceph-mon[25682]: pgmap v416: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.9 MiB/s wr, 558 op/s 2024-09-20T18:32:57.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 143453 10.12 MB/sec execute 228 sec latency 78.817 ms 2024-09-20T18:32:58.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 143748 10.12 MB/sec execute 229 sec latency 112.658 ms 2024-09-20T18:32:58.362 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:58 smithi012 ceph-mon[23745]: pgmap v417: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-09-20T18:32:58.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:58 smithi174 ceph-mon[25682]: pgmap v417: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-09-20T18:32:59.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 144101 10.13 MB/sec execute 230 sec latency 73.072 ms 2024-09-20T18:32:59.388 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:32:59.388 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:32:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:32:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:32:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:32:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:33:00.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 144468 10.13 MB/sec execute 231 sec latency 93.873 ms 2024-09-20T18:33:00.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:00 smithi012 ceph-mon[23745]: pgmap v418: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-20T18:33:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:00 smithi174 ceph-mon[25682]: pgmap v418: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 468 op/s 2024-09-20T18:33:01.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 144763 10.11 MB/sec execute 232 sec latency 87.005 ms 2024-09-20T18:33:01.553 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:33:02.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 145074 10.11 MB/sec execute 233 sec latency 40.112 ms 2024-09-20T18:33:02.149 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:01 smithi012 ceph-mon[23745]: pgmap v419: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 449 op/s 2024-09-20T18:33:02.149 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:01 smithi012 ceph-mon[23745]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:02.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:01 smithi174 ceph-mon[25682]: pgmap v419: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 449 op/s 2024-09-20T18:33:02.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:01 smithi174 ceph-mon[25682]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:02.514 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:33:02.514 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (11m) 5m ago 12m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:33:02.514 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (12m) 5m ago 12m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:33:02.514 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (11m) 5m ago 11m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (10m) 5m ago 12m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (6m) 5m ago 6m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (6m) 5m ago 6m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (13m) 5m ago 13m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (11m) 5m ago 11m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (13m) 5m ago 13m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (11m) 5m ago 11m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (6m) 5m ago 6m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (6m) 5m ago 6m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (12m) 5m ago 12m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (11m) 5m ago 11m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (10m) 5m ago 10m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:33:02.515 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (10m) 5m ago 10m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:33:02.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (9m) 5m ago 9m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:33:02.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (9m) 5m ago 9m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:33:02.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (9m) 5m ago 9m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:33:02.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (8m) 5m ago 8m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:33:02.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (8m) 5m ago 8m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:33:02.516 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (7m) 5m ago 7m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:33:02.516 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (11m) 5m ago 11m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:33:03.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 145587 10.13 MB/sec execute 234 sec latency 35.123 ms 2024-09-20T18:33:03.162 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:33:03.162 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:33:03.162 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:33:03.162 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:03.162 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:33:03.162 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:33:03.163 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:33:03.178 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:02 smithi012 ceph-mon[23745]: from='client.24571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:03.178 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:02 smithi012 ceph-mon[23745]: from='client.14852 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:03.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:02 smithi174 ceph-mon[25682]: from='client.24571 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:03.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:02 smithi174 ceph-mon[25682]: from='client.14852 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:03.687 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:33:03.687 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5", 2024-09-20T18:33:03.687 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:33:03.687 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:33:03.687 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "", 2024-09-20T18:33:03.687 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:33:03.687 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:33:04.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 145960 10.15 MB/sec execute 235 sec latency 50.865 ms 2024-09-20T18:33:04.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:03 smithi012 ceph-mon[23745]: pgmap v420: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 627 op/s 2024-09-20T18:33:04.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:03 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1583024023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:33:04.191 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:03 smithi012 ceph-mon[23745]: from='client.14860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:04.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:03 smithi174 ceph-mon[25682]: pgmap v420: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 627 op/s 2024-09-20T18:33:04.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:03 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1583024023' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:33:04.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:03 smithi174 ceph-mon[25682]: from='client.14860 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:04.277 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:33:05.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 146310 10.14 MB/sec execute 236 sec latency 123.243 ms 2024-09-20T18:33:05.203 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:04 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3114593859' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:33:05.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:04 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3114593859' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:33:06.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 146717 10.12 MB/sec execute 237 sec latency 64.735 ms 2024-09-20T18:33:06.211 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:05 smithi012 ceph-mon[23745]: pgmap v421: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 500 op/s 2024-09-20T18:33:06.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:05 smithi174 ceph-mon[25682]: pgmap v421: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 500 op/s 2024-09-20T18:33:07.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 146965 10.09 MB/sec execute 238 sec latency 51.744 ms 2024-09-20T18:33:08.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 147325 10.10 MB/sec execute 239 sec latency 74.035 ms 2024-09-20T18:33:08.263 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:08 smithi012 ceph-mon[23745]: pgmap v422: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 416 op/s 2024-09-20T18:33:08.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:08 smithi174 ceph-mon[25682]: pgmap v422: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 416 op/s 2024-09-20T18:33:09.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 147646 10.10 MB/sec execute 240 sec latency 52.469 ms 2024-09-20T18:33:10.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 147980 10.10 MB/sec execute 241 sec latency 58.762 ms 2024-09-20T18:33:10.324 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:10 smithi012 ceph-mon[23745]: pgmap v423: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-20T18:33:10.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:10 smithi174 ceph-mon[25682]: pgmap v423: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.2 MiB/s wr, 499 op/s 2024-09-20T18:33:11.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 148260 10.09 MB/sec execute 242 sec latency 65.474 ms 2024-09-20T18:33:12.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 148567 10.09 MB/sec execute 243 sec latency 37.119 ms 2024-09-20T18:33:12.122 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:11 smithi012 ceph-mon[23745]: pgmap v424: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.9 MiB/s wr, 454 op/s 2024-09-20T18:33:12.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:11 smithi174 ceph-mon[25682]: pgmap v424: 97 pgs: 97 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.9 MiB/s wr, 454 op/s 2024-09-20T18:33:13.094 INFO:teuthology.orchestra.run.smithi012.stdout: 5 148907 10.08 MB/sec execute 244 sec latency 64.736 ms 2024-09-20T18:33:14.104 INFO:teuthology.orchestra.run.smithi012.stdout: 5 149314 10.09 MB/sec execute 245 sec latency 54.782 ms 2024-09-20T18:33:14.195 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:14 smithi012 ceph-mon[23745]: pgmap v425: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 585 op/s 2024-09-20T18:33:14.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:14 smithi174 ceph-mon[25682]: pgmap v425: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 585 op/s 2024-09-20T18:33:15.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 149918 10.13 MB/sec execute 246 sec latency 37.042 ms 2024-09-20T18:33:16.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 150374 10.11 MB/sec execute 247 sec latency 34.061 ms 2024-09-20T18:33:16.206 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:16 smithi012 ceph-mon[23745]: pgmap v426: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 393 op/s 2024-09-20T18:33:16.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:16 smithi174 ceph-mon[25682]: pgmap v426: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 393 op/s 2024-09-20T18:33:17.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 150808 10.11 MB/sec execute 248 sec latency 36.892 ms 2024-09-20T18:33:18.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 151273 10.12 MB/sec execute 249 sec latency 47.434 ms 2024-09-20T18:33:18.222 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:18 smithi012 ceph-mon[23745]: pgmap v427: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 448 op/s 2024-09-20T18:33:18.320 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:18 smithi174 ceph-mon[25682]: pgmap v427: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 448 op/s 2024-09-20T18:33:19.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 151718 10.13 MB/sec execute 250 sec latency 53.373 ms 2024-09-20T18:33:20.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 152061 10.14 MB/sec execute 251 sec latency 43.027 ms 2024-09-20T18:33:20.240 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:20 smithi012 ceph-mon[23745]: pgmap v428: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 597 op/s 2024-09-20T18:33:20.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:20 smithi174 ceph-mon[25682]: pgmap v428: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 597 op/s 2024-09-20T18:33:21.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 152474 10.15 MB/sec execute 252 sec latency 35.126 ms 2024-09-20T18:33:22.003 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:21 smithi012 ceph-mon[23745]: pgmap v429: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 508 op/s 2024-09-20T18:33:22.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 153008 10.17 MB/sec execute 253 sec latency 53.476 ms 2024-09-20T18:33:22.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:21 smithi174 ceph-mon[25682]: pgmap v429: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.3 MiB/s wr, 508 op/s 2024-09-20T18:33:23.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 153360 10.17 MB/sec execute 254 sec latency 57.262 ms 2024-09-20T18:33:24.105 INFO:teuthology.orchestra.run.smithi012.stdout: 5 153659 10.15 MB/sec execute 255 sec latency 78.349 ms 2024-09-20T18:33:24.280 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:24 smithi012 ceph-mon[23745]: pgmap v430: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 650 op/s 2024-09-20T18:33:24.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:24 smithi174 ceph-mon[25682]: pgmap v430: 97 pgs: 97 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.8 MiB/s wr, 650 op/s 2024-09-20T18:33:25.106 INFO:teuthology.orchestra.run.smithi012.stdout: 5 153933 10.12 MB/sec execute 256 sec latency 62.935 ms 2024-09-20T18:33:26.106 INFO:teuthology.orchestra.run.smithi012.stdout: 5 154196 10.11 MB/sec execute 257 sec latency 51.367 ms 2024-09-20T18:33:26.354 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:26 smithi012 ceph-mon[23745]: pgmap v431: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 488 op/s 2024-09-20T18:33:26.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:26 smithi174 ceph-mon[25682]: pgmap v431: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 488 op/s 2024-09-20T18:33:27.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 154663 10.12 MB/sec execute 258 sec latency 38.987 ms 2024-09-20T18:33:28.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 155199 10.15 MB/sec execute 259 sec latency 41.380 ms 2024-09-20T18:33:28.374 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:28 smithi012 ceph-mon[23745]: pgmap v432: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.6 MiB/s wr, 676 op/s 2024-09-20T18:33:28.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:28 smithi174 ceph-mon[25682]: pgmap v432: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.6 MiB/s wr, 676 op/s 2024-09-20T18:33:29.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 155803 10.17 MB/sec execute 260 sec latency 24.451 ms 2024-09-20T18:33:30.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 156202 10.18 MB/sec execute 261 sec latency 53.535 ms 2024-09-20T18:33:30.307 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:30 smithi012 ceph-mon[23745]: pgmap v433: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 720 op/s 2024-09-20T18:33:30.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:30 smithi174 ceph-mon[25682]: pgmap v433: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 720 op/s 2024-09-20T18:33:31.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 156486 10.18 MB/sec execute 262 sec latency 68.578 ms 2024-09-20T18:33:32.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 156750 10.18 MB/sec execute 263 sec latency 69.589 ms 2024-09-20T18:33:32.130 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:31 smithi012 ceph-mon[23745]: pgmap v434: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 526 op/s 2024-09-20T18:33:32.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:31 smithi174 ceph-mon[25682]: pgmap v434: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.7 MiB/s wr, 526 op/s 2024-09-20T18:33:33.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 157271 10.17 MB/sec execute 264 sec latency 38.686 ms 2024-09-20T18:33:34.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 157721 10.16 MB/sec execute 265 sec latency 32.323 ms 2024-09-20T18:33:34.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:34 smithi012 ceph-mon[23745]: pgmap v435: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 728 op/s 2024-09-20T18:33:34.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:34 smithi174 ceph-mon[25682]: pgmap v435: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 728 op/s 2024-09-20T18:33:34.776 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:33:35.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 158015 10.15 MB/sec execute 266 sec latency 76.446 ms 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (11m) 6m ago 13m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (13m) 6m ago 12m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (12m) 6m ago 12m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (11m) 6m ago 12m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (7m) 6m ago 7m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (6m) 6m ago 6m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (14m) 6m ago 14m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (12m) 6m ago 12m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (14m) 6m ago 14m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (12m) 6m ago 12m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (6m) 6m ago 6m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:33:35.803 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (6m) 6m ago 6m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (12m) 6m ago 12m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (12m) 6m ago 12m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (11m) 6m ago 11m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (10m) 6m ago 10m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (10m) 6m ago 10m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (10m) 6m ago 10m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (9m) 6m ago 9m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (9m) 6m ago 9m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (8m) 6m ago 8m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (8m) 6m ago 8m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:33:35.804 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (11m) 6m ago 11m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:33:36.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 158319 10.16 MB/sec execute 267 sec latency 133.757 ms 2024-09-20T18:33:36.212 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:36 smithi012 ceph-mon[23745]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:36.212 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:36 smithi012 ceph-mon[23745]: pgmap v436: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.3 MiB/s wr, 567 op/s 2024-09-20T18:33:36.212 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:36 smithi012 ceph-mon[23745]: from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:33:36.351 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:33:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:33:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:33:36.352 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:33:36.352 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:33:36.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:36 smithi174 ceph-mon[25682]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:36.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:36 smithi174 ceph-mon[25682]: pgmap v436: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.3 MiB/s wr, 567 op/s 2024-09-20T18:33:36.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:36 smithi174 ceph-mon[25682]: from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:36.830 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:33:36.830 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:33:36.830 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:33:36.830 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:33:36.830 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "0/23 daemons upgraded", 2024-09-20T18:33:36.830 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:88deadb8082f4a33832e228ccefc59cedba158a5 image" 2024-09-20T18:33:36.830 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:33:37.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 158646 10.15 MB/sec execute 268 sec latency 144.785 ms 2024-09-20T18:33:37.237 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:37 smithi012 ceph-mon[23745]: from='client.24593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:37.237 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:37 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1037442825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:33:37.237 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:33:37.237 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:33:37.237 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:37 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:33:37.395 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:33:37.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:37 smithi174 ceph-mon[25682]: from='client.24593 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:37.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:37 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1037442825' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:33:37.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:33:37.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:33:37.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:37 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:33:38.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 158999 10.15 MB/sec execute 269 sec latency 53.937 ms 2024-09-20T18:33:38.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:38 smithi012 ceph-mon[23745]: Upgrade: Target is version 19.3.0-5099-g88deadb8 (unknown) 2024-09-20T18:33:38.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:38 smithi012 ceph-mon[23745]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2'] 2024-09-20T18:33:38.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:38 smithi012 ceph-mon[23745]: from='client.14884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:38.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:38 smithi012 ceph-mon[23745]: Upgrade: Need to upgrade myself (mgr.smithi012.xkxxrd) 2024-09-20T18:33:38.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:38 smithi012 ceph-mon[23745]: pgmap v437: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.8 MiB/s wr, 604 op/s 2024-09-20T18:33:38.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:38 smithi012 ceph-mon[23745]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 on smithi174 2024-09-20T18:33:38.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:38 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/52735528' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:33:38.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:38 smithi174 ceph-mon[25682]: Upgrade: Target is version 19.3.0-5099-g88deadb8 (unknown) 2024-09-20T18:33:38.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:38 smithi174 ceph-mon[25682]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, digests ['quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2'] 2024-09-20T18:33:38.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:38 smithi174 ceph-mon[25682]: from='client.14884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:33:38.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:38 smithi174 ceph-mon[25682]: Upgrade: Need to upgrade myself (mgr.smithi012.xkxxrd) 2024-09-20T18:33:38.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:38 smithi174 ceph-mon[25682]: pgmap v437: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.8 MiB/s wr, 604 op/s 2024-09-20T18:33:38.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:38 smithi174 ceph-mon[25682]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 on smithi174 2024-09-20T18:33:38.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:38 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/52735528' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:33:39.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 159241 10.14 MB/sec execute 270 sec latency 64.912 ms 2024-09-20T18:33:40.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 159498 10.14 MB/sec execute 271 sec latency 48.670 ms 2024-09-20T18:33:40.316 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:40 smithi012 ceph-mon[23745]: pgmap v438: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 549 op/s 2024-09-20T18:33:40.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:40 smithi174 ceph-mon[25682]: pgmap v438: 97 pgs: 97 active+clean; 344 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 549 op/s 2024-09-20T18:33:41.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 159752 10.13 MB/sec execute 272 sec latency 58.950 ms 2024-09-20T18:33:42.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 160014 10.12 MB/sec execute 273 sec latency 67.807 ms 2024-09-20T18:33:42.110 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:41 smithi012 ceph-mon[23745]: pgmap v439: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 459 op/s 2024-09-20T18:33:42.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:41 smithi174 ceph-mon[25682]: pgmap v439: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.0 MiB/s wr, 459 op/s 2024-09-20T18:33:43.107 INFO:teuthology.orchestra.run.smithi012.stdout: 5 160510 10.13 MB/sec execute 274 sec latency 47.673 ms 2024-09-20T18:33:44.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 160798 10.11 MB/sec execute 275 sec latency 68.586 ms 2024-09-20T18:33:44.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:44 smithi012 ceph-mon[23745]: pgmap v440: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.0 MiB/s wr, 582 op/s 2024-09-20T18:33:44.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:44 smithi174 ceph-mon[25682]: pgmap v440: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.0 MiB/s wr, 582 op/s 2024-09-20T18:33:45.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 161045 10.09 MB/sec execute 276 sec latency 57.273 ms 2024-09-20T18:33:46.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 161509 10.09 MB/sec execute 277 sec latency 57.655 ms 2024-09-20T18:33:46.198 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:46 smithi012 ceph-mon[23745]: pgmap v441: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 359 op/s 2024-09-20T18:33:46.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:46 smithi174 ceph-mon[25682]: pgmap v441: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 359 op/s 2024-09-20T18:33:47.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 162288 10.14 MB/sec execute 278 sec latency 32.814 ms 2024-09-20T18:33:48.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 162896 10.16 MB/sec execute 279 sec latency 30.448 ms 2024-09-20T18:33:48.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:48 smithi012 ceph-mon[23745]: pgmap v442: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 537 op/s 2024-09-20T18:33:48.321 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:48 smithi174 ceph-mon[25682]: pgmap v442: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.1 MiB/s wr, 537 op/s 2024-09-20T18:33:49.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 163397 10.19 MB/sec execute 280 sec latency 32.979 ms 2024-09-20T18:33:50.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 163711 10.19 MB/sec execute 281 sec latency 57.184 ms 2024-09-20T18:33:50.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:50 smithi012 ceph-mon[23745]: pgmap v443: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.4 MiB/s wr, 545 op/s 2024-09-20T18:33:50.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:50 smithi174 ceph-mon[25682]: pgmap v443: 97 pgs: 97 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.4 MiB/s wr, 545 op/s 2024-09-20T18:33:51.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 163974 10.17 MB/sec execute 282 sec latency 108.331 ms 2024-09-20T18:33:51.993 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:51 smithi012 ceph-mon[23745]: pgmap v444: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 429 op/s 2024-09-20T18:33:52.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 164183 10.15 MB/sec execute 283 sec latency 50.912 ms 2024-09-20T18:33:52.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:51 smithi174 ceph-mon[25682]: pgmap v444: 97 pgs: 97 active+clean; 346 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.0 MiB/s wr, 429 op/s 2024-09-20T18:33:53.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 164404 10.13 MB/sec execute 284 sec latency 63.625 ms 2024-09-20T18:33:54.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 164773 10.13 MB/sec execute 285 sec latency 59.392 ms 2024-09-20T18:33:54.276 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:54 smithi012 ceph-mon[23745]: pgmap v445: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-20T18:33:54.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:54 smithi174 ceph-mon[25682]: pgmap v445: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.6 MiB/s wr, 654 op/s 2024-09-20T18:33:55.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 165072 10.11 MB/sec execute 286 sec latency 66.804 ms 2024-09-20T18:33:56.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 165377 10.11 MB/sec execute 287 sec latency 48.978 ms 2024-09-20T18:33:56.305 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:56 smithi012 ceph-mon[23745]: pgmap v446: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 567 op/s 2024-09-20T18:33:56.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:56 smithi174 ceph-mon[25682]: pgmap v446: 97 pgs: 97 active+clean; 353 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.4 MiB/s wr, 567 op/s 2024-09-20T18:33:57.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 165687 10.11 MB/sec execute 288 sec latency 44.208 ms 2024-09-20T18:33:58.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 166048 10.10 MB/sec execute 289 sec latency 58.447 ms 2024-09-20T18:33:58.359 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:58 smithi012 ceph-mon[23745]: pgmap v447: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.8 MiB/s wr, 620 op/s 2024-09-20T18:33:58.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:58 smithi174 ceph-mon[25682]: pgmap v447: 97 pgs: 97 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.8 MiB/s wr, 620 op/s 2024-09-20T18:33:59.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 166345 10.10 MB/sec execute 290 sec latency 51.515 ms 2024-09-20T18:33:59.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:33:59.370 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:33:59 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:33:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:33:59.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:33:59 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:34:00.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 166607 10.10 MB/sec execute 291 sec latency 36.438 ms 2024-09-20T18:34:00.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:00 smithi012 ceph-mon[23745]: pgmap v448: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.1 MiB/s wr, 466 op/s 2024-09-20T18:34:00.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:00 smithi174 ceph-mon[25682]: pgmap v448: 97 pgs: 97 active+clean; 344 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.1 MiB/s wr, 466 op/s 2024-09-20T18:34:01.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 166870 10.09 MB/sec execute 292 sec latency 47.933 ms 2024-09-20T18:34:02.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 167210 10.10 MB/sec execute 293 sec latency 45.576 ms 2024-09-20T18:34:02.187 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:01 smithi012 ceph-mon[23745]: pgmap v449: 97 pgs: 97 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 471 op/s 2024-09-20T18:34:02.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:01 smithi174 ceph-mon[25682]: pgmap v449: 97 pgs: 97 active+clean; 346 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.8 MiB/s wr, 471 op/s 2024-09-20T18:34:03.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 167525 10.08 MB/sec execute 294 sec latency 51.845 ms 2024-09-20T18:34:04.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 167766 10.06 MB/sec execute 295 sec latency 48.509 ms 2024-09-20T18:34:04.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:04 smithi012 ceph-mon[23745]: pgmap v450: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-20T18:34:04.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:04 smithi174 ceph-mon[25682]: pgmap v450: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.6 MiB/s wr, 593 op/s 2024-09-20T18:34:05.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 167962 10.05 MB/sec execute 296 sec latency 131.690 ms 2024-09-20T18:34:06.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 168170 10.02 MB/sec execute 297 sec latency 93.945 ms 2024-09-20T18:34:06.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:06 smithi012 ceph-mon[23745]: pgmap v451: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 357 op/s 2024-09-20T18:34:06.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:06 smithi174 ceph-mon[25682]: pgmap v451: 97 pgs: 97 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.8 MiB/s wr, 357 op/s 2024-09-20T18:34:07.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 168474 10.02 MB/sec execute 298 sec latency 66.810 ms 2024-09-20T18:34:07.854 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:34:08.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 168744 10.00 MB/sec execute 299 sec latency 41.338 ms 2024-09-20T18:34:08.314 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:08 smithi012 ceph-mon[23745]: pgmap v452: 97 pgs: 97 active+clean; 355 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 453 op/s 2024-09-20T18:34:08.382 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:08 smithi174 ceph-mon[25682]: pgmap v452: 97 pgs: 97 active+clean; 355 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 453 op/s 2024-09-20T18:34:08.791 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (12m) 7m ago 13m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (13m) 7m ago 13m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (13m) 7m ago 13m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (11m) 7m ago 13m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (7m) 7m ago 7m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (7m) 7m ago 7m 12.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (14m) 7m ago 14m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (12m) 7m ago 12m 414M - 17.2.0 e1d6a67b021e 59e693d47a82 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (14m) 7m ago 14m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (12m) 7m ago 12m 37.4M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (7m) 7m ago 7m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:34:08.792 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (7m) 7m ago 7m 48.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (13m) 7m ago 13m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (12m) 7m ago 12m 18.1M - 1dbe0e931976 1c69bbee0c99 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (12m) 7m ago 12m 45.5M 3217M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (11m) 7m ago 11m 42.5M 3217M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (11m) 7m ago 11m 41.2M 3217M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (10m) 7m ago 10m 39.5M 3217M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (10m) 7m ago 10m 39.1M 3984M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (9m) 7m ago 9m 40.4M 3984M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (9m) 7m ago 9m 40.1M 3984M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (9m) 7m ago 9m 40.9M 3984M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:34:08.793 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (12m) 7m ago 12m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:34:09.108 INFO:teuthology.orchestra.run.smithi012.stdout: 5 169111 10.01 MB/sec execute 300 sec latency 54.300 ms 2024-09-20T18:34:09.397 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:09.398 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:34:09.399 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-20T18:34:09.399 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:34:09.399 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: from='client.24611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: Upgrade: Updating mgr.smithi174.nxkptu 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: Deploying daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:34:09.590 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:09 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2752287357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: from='client.14892 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: from='client.24611 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: Upgrade: Updating mgr.smithi174.nxkptu 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: Deploying daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:34:09.690 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:09 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2752287357' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:34:09.930 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:34:09.930 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:34:09.930 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:34:09.930 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:34:09.930 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "0/23 daemons upgraded", 2024-09-20T18:34:09.930 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading mgr daemons" 2024-09-20T18:34:09.930 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:34:10.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 169482 10.01 MB/sec execute 301 sec latency 64.575 ms 2024-09-20T18:34:10.562 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:34:10.599 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:10 smithi012 ceph-mon[23745]: from='client.24615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:10.599 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:10 smithi012 ceph-mon[23745]: pgmap v453: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.4 MiB/s wr, 420 op/s 2024-09-20T18:34:10.681 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:10 smithi174 ceph-mon[25682]: from='client.24615 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:10.681 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:10 smithi174 ceph-mon[25682]: pgmap v453: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.4 MiB/s wr, 420 op/s 2024-09-20T18:34:11.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 169883 10.01 MB/sec execute 302 sec latency 44.540 ms 2024-09-20T18:34:11.424 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:11 smithi174 ceph-mon[25682]: from='client.14906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:11.618 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:11 smithi012 ceph-mon[23745]: from='client.14906 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:11.618 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:11 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1252722088' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:34:11.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:11 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1252722088' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:34:12.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 170178 10.02 MB/sec execute 303 sec latency 42.749 ms 2024-09-20T18:34:12.658 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:12 smithi012 ceph-mon[23745]: pgmap v454: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 408 op/s 2024-09-20T18:34:12.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:12 smithi174 ceph-mon[25682]: pgmap v454: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 408 op/s 2024-09-20T18:34:13.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 170397 10.01 MB/sec execute 304 sec latency 100.085 ms 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: pgmap v455: 97 pgs: 97 active+clean; 357 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:13.973 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:13.974 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:13.974 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi012 to 1937M 2024-09-20T18:34:13.974 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:13 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:34:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:34:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:34:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: pgmap v455: 97 pgs: 97 active+clean; 357 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-09-20T18:34:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:14.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:14.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:14.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:14.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:14.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi012 to 1937M 2024-09-20T18:34:14.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:13 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:14.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 170665 10.01 MB/sec execute 305 sec latency 84.316 ms 2024-09-20T18:34:15.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 170939 9.99 MB/sec execute 306 sec latency 49.716 ms 2024-09-20T18:34:16.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 171225 9.97 MB/sec execute 307 sec latency 49.691 ms 2024-09-20T18:34:16.282 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:16 smithi012 ceph-mon[23745]: pgmap v456: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 448 op/s 2024-09-20T18:34:16.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:16 smithi174 ceph-mon[25682]: pgmap v456: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 448 op/s 2024-09-20T18:34:17.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 171474 9.96 MB/sec execute 308 sec latency 47.422 ms 2024-09-20T18:34:18.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 171771 9.95 MB/sec execute 309 sec latency 64.395 ms 2024-09-20T18:34:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:18 smithi012 ceph-mon[23745]: pgmap v457: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 467 op/s 2024-09-20T18:34:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:18 smithi012 ceph-mon[23745]: Standby manager daemon smithi174.nxkptu restarted 2024-09-20T18:34:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:18 smithi012 ceph-mon[23745]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:34:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:18 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:34:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:18 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:34:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:18 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:34:18.319 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:18 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:34:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:18 smithi174 ceph-mon[25682]: pgmap v457: 97 pgs: 97 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 467 op/s 2024-09-20T18:34:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:18 smithi174 ceph-mon[25682]: Standby manager daemon smithi174.nxkptu restarted 2024-09-20T18:34:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:18 smithi174 ceph-mon[25682]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:34:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:18 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:34:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:18 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:34:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:18 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:34:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:18 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:34:19.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 172061 9.94 MB/sec execute 310 sec latency 54.210 ms 2024-09-20T18:34:19.358 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:19 smithi012 ceph-mon[23745]: mgrmap e22: smithi012.xkxxrd(active, since 13m), standbys: smithi174.nxkptu 2024-09-20T18:34:19.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:19 smithi174 ceph-mon[25682]: mgrmap e22: smithi012.xkxxrd(active, since 13m), standbys: smithi174.nxkptu 2024-09-20T18:34:20.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 172603 9.96 MB/sec execute 311 sec latency 49.546 ms 2024-09-20T18:34:20.366 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:20 smithi012 ceph-mon[23745]: pgmap v458: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.1 MiB/s wr, 377 op/s 2024-09-20T18:34:20.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:20 smithi174 ceph-mon[25682]: pgmap v458: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.1 MiB/s wr, 377 op/s 2024-09-20T18:34:21.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 173170 9.98 MB/sec execute 312 sec latency 33.624 ms 2024-09-20T18:34:22.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 173667 9.99 MB/sec execute 313 sec latency 32.404 ms 2024-09-20T18:34:22.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:21 smithi012 ceph-mon[23745]: pgmap v459: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 407 op/s 2024-09-20T18:34:22.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:21 smithi174 ceph-mon[25682]: pgmap v459: 97 pgs: 97 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.2 MiB/s wr, 407 op/s 2024-09-20T18:34:23.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 174193 10.02 MB/sec execute 314 sec latency 42.350 ms 2024-09-20T18:34:24.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 174701 10.02 MB/sec execute 315 sec latency 32.544 ms 2024-09-20T18:34:24.437 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:24 smithi012 ceph-mon[23745]: pgmap v460: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 536 op/s 2024-09-20T18:34:24.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:24 smithi174 ceph-mon[25682]: pgmap v460: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 536 op/s 2024-09-20T18:34:25.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 175101 10.02 MB/sec execute 316 sec latency 31.517 ms 2024-09-20T18:34:26.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 175579 10.02 MB/sec execute 317 sec latency 77.278 ms 2024-09-20T18:34:26.208 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:26 smithi012 ceph-mon[23745]: pgmap v461: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.7 MiB/s wr, 481 op/s 2024-09-20T18:34:26.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:26 smithi174 ceph-mon[25682]: pgmap v461: 97 pgs: 97 active+clean; 353 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.7 MiB/s wr, 481 op/s 2024-09-20T18:34:27.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 176101 10.04 MB/sec execute 318 sec latency 35.526 ms 2024-09-20T18:34:28.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 176551 10.04 MB/sec execute 319 sec latency 43.287 ms 2024-09-20T18:34:28.232 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:28 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:28.232 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:28 smithi012 ceph-mon[23745]: pgmap v462: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 457 op/s 2024-09-20T18:34:28.232 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:28 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:28.328 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:28 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:28.328 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:28 smithi174 ceph-mon[25682]: pgmap v462: 97 pgs: 97 active+clean; 348 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.2 MiB/s wr, 457 op/s 2024-09-20T18:34:28.329 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:28 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:29.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 177005 10.05 MB/sec execute 320 sec latency 32.780 ms 2024-09-20T18:34:30.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 177314 10.06 MB/sec execute 321 sec latency 42.361 ms 2024-09-20T18:34:30.268 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:30 smithi012 ceph-mon[23745]: pgmap v463: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 592 op/s 2024-09-20T18:34:30.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:30 smithi174 ceph-mon[25682]: pgmap v463: 97 pgs: 97 active+clean; 347 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 592 op/s 2024-09-20T18:34:31.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 177644 10.06 MB/sec execute 322 sec latency 73.725 ms 2024-09-20T18:34:32.043 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: Upgrade: Need to upgrade myself (mgr.smithi012.xkxxrd) 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: Failing over to other MGR 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr fail", "who": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: pgmap v465: 97 pgs: 97 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.9 MiB/s wr, 652 op/s 2024-09-20T18:34:32.044 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:31 smithi012 ceph-mon[23745]: Standby manager daemon smithi012.xkxxrd started 2024-09-20T18:34:32.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 178007 10.05 MB/sec execute 323 sec latency 37.132 ms 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: Upgrade: Need to upgrade myself (mgr.smithi012.xkxxrd) 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: Failing over to other MGR 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr fail", "who": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: osdmap e71: 8 total, 8 up, 8 in 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: pgmap v465: 97 pgs: 97 active+clean; 352 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.9 MiB/s wr, 652 op/s 2024-09-20T18:34:32.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:31 smithi174 ceph-mon[25682]: Standby manager daemon smithi012.xkxxrd started 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "mgr fail", "who": "smithi012.xkxxrd"}]': finished 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: mgrmap e23: smithi174.nxkptu(active, starting, since 0.999931s), standbys: smithi012.xkxxrd 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"}]: dispatch 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"}]: dispatch 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr metadata", "who": "smithi174.nxkptu", "id": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:34:33.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: Manager daemon smithi174.nxkptu is now available 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.052 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.053 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.053 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: Checking for cert/key for grafana.smithi012 2024-09-20T18:34:33.053 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.053 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:34:33.053 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:32 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:34:33.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 178287 10.04 MB/sec execute 324 sec latency 66.357 ms 2024-09-20T18:34:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:34:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:34:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.14202 172.21.15.12:0/3868969748' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "mgr fail", "who": "smithi012.xkxxrd"}]': finished 2024-09-20T18:34:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: mgrmap e23: smithi174.nxkptu(active, starting, since 0.999931s), standbys: smithi012.xkxxrd 2024-09-20T18:34:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"}]: dispatch 2024-09-20T18:34:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"}]: dispatch 2024-09-20T18:34:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr metadata", "who": "smithi174.nxkptu", "id": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: Manager daemon smithi174.nxkptu is now available 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: Checking for cert/key for grafana.smithi012 2024-09-20T18:34:33.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:33.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:34:33.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:32 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:34:34.075 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:33 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:34:34.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:33 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:34:34.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:33 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/trash_purge_schedule"}]: dispatch 2024-09-20T18:34:34.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:33 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/trash_purge_schedule"}]: dispatch 2024-09-20T18:34:34.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:33 smithi012 ceph-mon[23745]: mgrmap e24: smithi174.nxkptu(active, since 2s), standbys: smithi012.xkxxrd 2024-09-20T18:34:34.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:33 smithi012 ceph-mon[23745]: pgmap v3: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:34.076 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:33 smithi012 ceph-mon[23745]: Deploying cephadm binary to smithi174 2024-09-20T18:34:34.110 INFO:teuthology.orchestra.run.smithi012.stdout: 5 178575 10.03 MB/sec execute 325 sec latency 57.482 ms 2024-09-20T18:34:34.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:33 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:34:34.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:33 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:34:34.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:33 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/trash_purge_schedule"}]: dispatch 2024-09-20T18:34:34.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:33 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi174.nxkptu/trash_purge_schedule"}]: dispatch 2024-09-20T18:34:34.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:33 smithi174 ceph-mon[25682]: mgrmap e24: smithi174.nxkptu(active, since 2s), standbys: smithi012.xkxxrd 2024-09-20T18:34:34.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:33 smithi174 ceph-mon[25682]: pgmap v3: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:34.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:33 smithi174 ceph-mon[25682]: Deploying cephadm binary to smithi174 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: pgmap v4: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: mgrmap e25: smithi174.nxkptu(active, since 3s), standbys: smithi012.xkxxrd 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: [20/Sep/2024:18:34:34] ENGINE Bus STARTING 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: Deploying cephadm binary to smithi012 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: [20/Sep/2024:18:34:34] ENGINE Serving on https://172.21.15.174:7150 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: [20/Sep/2024:18:34:34] ENGINE Error in HTTPServer.serve 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Traceback (most recent call last): 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T18:34:35.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._connections.run(self.expiration_interval) 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._run(expiration_interval) 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: s = self.context.wrap_socket( 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: return self.sslsocket_class._create( 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self.do_handshake() 2024-09-20T18:34:35.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T18:34:35.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._sslobj.do_handshake() 2024-09-20T18:34:35.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T18:34:35.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: [20/Sep/2024:18:34:34] ENGINE Serving on http://172.21.15.174:8765 2024-09-20T18:34:35.070 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:34 smithi012 ceph-mon[23745]: [20/Sep/2024:18:34:34] ENGINE Bus STARTED 2024-09-20T18:34:35.109 INFO:teuthology.orchestra.run.smithi012.stdout: 5 178875 10.02 MB/sec execute 326 sec latency 53.560 ms 2024-09-20T18:34:35.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: pgmap v4: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:35.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: mgrmap e25: smithi174.nxkptu(active, since 3s), standbys: smithi012.xkxxrd 2024-09-20T18:34:35.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: [20/Sep/2024:18:34:34] ENGINE Bus STARTING 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: Deploying cephadm binary to smithi012 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: [20/Sep/2024:18:34:34] ENGINE Serving on https://172.21.15.174:7150 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: [20/Sep/2024:18:34:34] ENGINE Error in HTTPServer.serve 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Traceback (most recent call last): 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._connections.run(self.expiration_interval) 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._run(expiration_interval) 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T18:34:35.258 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: s = self.context.wrap_socket( 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: return self.sslsocket_class._create( 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self.do_handshake() 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._sslobj.do_handshake() 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: [20/Sep/2024:18:34:34] ENGINE Serving on http://172.21.15.174:8765 2024-09-20T18:34:35.259 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:34 smithi174 ceph-mon[25682]: [20/Sep/2024:18:34:34] ENGINE Bus STARTED 2024-09-20T18:34:36.110 INFO:teuthology.orchestra.run.smithi012.stdout: 5 179226 10.02 MB/sec execute 327 sec latency 56.611 ms 2024-09-20T18:34:37.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:37 smithi012 ceph-mon[23745]: pgmap v5: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:37.082 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:37 smithi012 ceph-mon[23745]: mgrmap e26: smithi174.nxkptu(active, since 5s), standbys: smithi012.xkxxrd 2024-09-20T18:34:37.110 INFO:teuthology.orchestra.run.smithi012.stdout: 5 179691 10.03 MB/sec execute 328 sec latency 66.073 ms 2024-09-20T18:34:37.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:37 smithi174 ceph-mon[25682]: pgmap v5: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:37.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:37 smithi174 ceph-mon[25682]: mgrmap e26: smithi174.nxkptu(active, since 5s), standbys: smithi012.xkxxrd 2024-09-20T18:34:38.110 INFO:teuthology.orchestra.run.smithi012.stdout: 5 180020 10.02 MB/sec execute 329 sec latency 61.790 ms 2024-09-20T18:34:39.111 INFO:teuthology.orchestra.run.smithi012.stdout: 5 180289 10.01 MB/sec execute 330 sec latency 67.686 ms 2024-09-20T18:34:39.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:39 smithi012 ceph-mon[23745]: pgmap v6: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:39.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:39 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:39.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:39 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:39.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:39 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:39.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:39 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:39.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:39 smithi174 ceph-mon[25682]: pgmap v6: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:34:39.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:39 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:39.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:39 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:39.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:39 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:39.393 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:39 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:40.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 180519 10.00 MB/sec execute 331 sec latency 49.278 ms 2024-09-20T18:34:41.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 180731 9.99 MB/sec execute 332 sec latency 66.450 ms 2024-09-20T18:34:41.140 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:34:41.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:41 smithi012 ceph-mon[23745]: pgmap v7: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 31 KiB/s rd, 3.5 MiB/s wr, 387 op/s 2024-09-20T18:34:41.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:41 smithi012 ceph-mon[23745]: Standby manager daemon smithi012.xkxxrd restarted 2024-09-20T18:34:41.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:41 smithi012 ceph-mon[23745]: Standby manager daemon smithi012.xkxxrd started 2024-09-20T18:34:41.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:41 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/crt"}]: dispatch 2024-09-20T18:34:41.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:41 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:34:41.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:41 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/key"}]: dispatch 2024-09-20T18:34:41.205 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:41 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:34:41.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:41 smithi174 ceph-mon[25682]: pgmap v7: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 31 KiB/s rd, 3.5 MiB/s wr, 387 op/s 2024-09-20T18:34:41.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:41 smithi174 ceph-mon[25682]: Standby manager daemon smithi012.xkxxrd restarted 2024-09-20T18:34:41.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:41 smithi174 ceph-mon[25682]: Standby manager daemon smithi012.xkxxrd started 2024-09-20T18:34:41.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:41 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/crt"}]: dispatch 2024-09-20T18:34:41.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:41 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:34:41.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:41 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/key"}]: dispatch 2024-09-20T18:34:41.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:41 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/277606664' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:34:42.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 180929 9.98 MB/sec execute 333 sec latency 49.775 ms 2024-09-20T18:34:42.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='client.14938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:42.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: mgrmap e27: smithi174.nxkptu(active, since 10s), standbys: smithi012.xkxxrd 2024-09-20T18:34:42.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='client.24657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:42.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:42.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:42.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.253 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:42 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (12m) 7m ago 14m 25.0M - ba2b418f427c f9d7e2615065 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (14m) 7m ago 14m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (13m) 4s ago 13m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (12m) 7m ago 13m 53.1M - 8.3.5 dad864ee21e9 49f978813096 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (8m) 7m ago 8m 16.0M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (8m) 4s ago 8m 14.3M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (15m) 7m ago 15m 460M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (30s) 4s ago 13m 495M - 19.3.0-5099-g88deadb8 441f9a3fe515 a5d2867ff390 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (15m) 7m ago 15m 48.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (13m) 4s ago 13m 45.8M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:34:42.257 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (7m) 7m ago 7m 49.8M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (7m) 4s ago 7m 83.4M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (13m) 7m ago 13m 17.7M - 1dbe0e931976 721d4388b70c 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (13m) 4s ago 13m 20.3M - 1.3.1 1dbe0e931976 1c69bbee0c99 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (12m) 7m ago 12m 45.5M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (12m) 7m ago 12m 42.5M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (11m) 7m ago 11m 41.2M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (11m) 7m ago 11m 39.5M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (10m) 4s ago 10m 544M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (10m) 4s ago 10m 402M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (10m) 4s ago 10m 577M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (9m) 4s ago 9m 478M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:34:42.258 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (13m) 7m ago 13m 63.2M - 514e6a882f6e ef74fb47ac7d 2024-09-20T18:34:42.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='client.14938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: mgrmap e27: smithi174.nxkptu(active, since 10s), standbys: smithi012.xkxxrd 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='client.24657 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-20T18:34:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:42 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:42.914 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:34:42.914 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:34:42.914 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:34:42.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:42.914 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:34:42.915 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:34:43.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 181220 9.98 MB/sec execute 334 sec latency 80.403 ms 2024-09-20T18:34:43.256 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:43 smithi012 ceph-mon[23745]: pgmap v8: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 MiB/s wr, 300 op/s 2024-09-20T18:34:43.256 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:43 smithi012 ceph-mon[23745]: Adjusting osd_memory_target on smithi174 to 2704M 2024-09-20T18:34:43.256 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:43 smithi012 ceph-mon[23745]: from='client.24659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:43.256 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:43 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/5247614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:34:43.257 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:43 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:43.257 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:43 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "", 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:34:43.500 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:34:43.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:43 smithi174 ceph-mon[25682]: pgmap v8: 97 pgs: 97 active+clean; 350 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 MiB/s wr, 300 op/s 2024-09-20T18:34:43.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:43 smithi174 ceph-mon[25682]: Adjusting osd_memory_target on smithi174 to 2704M 2024-09-20T18:34:43.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:43 smithi174 ceph-mon[25682]: from='client.24659 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:43.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:43 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/5247614' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:34:43.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:43 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:43.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:43 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:44.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 181563 9.98 MB/sec execute 335 sec latency 52.674 ms 2024-09-20T18:34:44.124 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:34:45.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='client.14954 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:45.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:34:45.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:34:45.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:34:45.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2963180091' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.059 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:44 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 181835 9.97 MB/sec execute 336 sec latency 62.171 ms 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='client.14954 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2963180091' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:44 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: pgmap v9: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:34:46.074 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Reconfiguring alertmanager.smithi012 (dependencies changed)... 2024-09-20T18:34:46.075 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:45 smithi012 ceph-mon[23745]: Deploying daemon alertmanager.smithi012 on smithi012 2024-09-20T18:34:46.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 182118 9.96 MB/sec execute 337 sec latency 55.551 ms 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: pgmap v9: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 22 KiB/s rd, 4.0 MiB/s wr, 405 op/s 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Reconfiguring alertmanager.smithi012 (dependencies changed)... 2024-09-20T18:34:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:45 smithi174 ceph-mon[25682]: Deploying daemon alertmanager.smithi012 on smithi012 2024-09-20T18:34:47.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 182450 9.95 MB/sec execute 338 sec latency 51.163 ms 2024-09-20T18:34:47.861 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:47 smithi012 ceph-mon[23745]: pgmap v10: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 3.6 MiB/s wr, 369 op/s 2024-09-20T18:34:48.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 182778 9.95 MB/sec execute 339 sec latency 42.912 ms 2024-09-20T18:34:48.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:47 smithi174 ceph-mon[25682]: pgmap v10: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 3.6 MiB/s wr, 369 op/s 2024-09-20T18:34:48.900 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:48 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:34:49.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 183049 9.93 MB/sec execute 340 sec latency 51.921 ms 2024-09-20T18:34:49.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:48 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:34:49.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:49 smithi012 ceph-mon[23745]: pgmap v11: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 3.6 MiB/s wr, 369 op/s 2024-09-20T18:34:50.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 183352 9.93 MB/sec execute 341 sec latency 48.451 ms 2024-09-20T18:34:50.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:49 smithi174 ceph-mon[25682]: pgmap v11: 97 pgs: 97 active+clean; 353 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 20 KiB/s rd, 3.6 MiB/s wr, 369 op/s 2024-09-20T18:34:51.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 183763 9.93 MB/sec execute 342 sec latency 57.756 ms 2024-09-20T18:34:51.940 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:51 smithi012 ceph-mon[23745]: pgmap v12: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 5.5 MiB/s wr, 561 op/s 2024-09-20T18:34:52.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 184094 9.93 MB/sec execute 343 sec latency 69.808 ms 2024-09-20T18:34:52.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:51 smithi174 ceph-mon[25682]: pgmap v12: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 5.5 MiB/s wr, 561 op/s 2024-09-20T18:34:53.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 184363 9.93 MB/sec execute 344 sec latency 68.767 ms 2024-09-20T18:34:54.048 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:53 smithi012 ceph-mon[23745]: pgmap v13: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 337 op/s 2024-09-20T18:34:54.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 184629 9.92 MB/sec execute 345 sec latency 75.607 ms 2024-09-20T18:34:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:53 smithi174 ceph-mon[25682]: pgmap v13: 97 pgs: 97 active+clean; 351 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 337 op/s 2024-09-20T18:34:55.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 185008 9.93 MB/sec execute 346 sec latency 104.585 ms 2024-09-20T18:34:56.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:55 smithi012 ceph-mon[23745]: pgmap v14: 97 pgs: 97 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 534 op/s 2024-09-20T18:34:56.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 185297 9.93 MB/sec execute 347 sec latency 85.694 ms 2024-09-20T18:34:56.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:55 smithi174 ceph-mon[25682]: pgmap v14: 97 pgs: 97 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 534 op/s 2024-09-20T18:34:57.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 185575 9.92 MB/sec execute 348 sec latency 62.931 ms 2024-09-20T18:34:58.106 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:57 smithi012 ceph-mon[23745]: pgmap v15: 97 pgs: 97 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-09-20T18:34:58.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 186015 9.91 MB/sec execute 349 sec latency 37.871 ms 2024-09-20T18:34:58.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:57 smithi174 ceph-mon[25682]: pgmap v15: 97 pgs: 97 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 388 op/s 2024-09-20T18:34:59.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 186348 9.91 MB/sec execute 350 sec latency 43.679 ms 2024-09-20T18:34:59.899 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:34:59 smithi012 ceph-mon[23745]: pgmap v16: 97 pgs: 97 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 389 op/s 2024-09-20T18:35:00.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 186732 9.92 MB/sec execute 351 sec latency 43.841 ms 2024-09-20T18:35:00.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:34:59 smithi174 ceph-mon[25682]: pgmap v16: 97 pgs: 97 active+clean; 350 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 389 op/s 2024-09-20T18:35:01.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 186947 9.90 MB/sec execute 352 sec latency 60.506 ms 2024-09-20T18:35:01.946 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:01 smithi012 ceph-mon[23745]: pgmap v17: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 584 op/s 2024-09-20T18:35:02.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 187181 9.88 MB/sec execute 353 sec latency 99.127 ms 2024-09-20T18:35:02.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:01 smithi174 ceph-mon[25682]: pgmap v17: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.8 MiB/s wr, 584 op/s 2024-09-20T18:35:03.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 187514 9.88 MB/sec execute 354 sec latency 60.852 ms 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: pgmap v18: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 392 op/s 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: Reconfiguring grafana.smithi012 (dependencies changed)... 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.511 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:03 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: pgmap v18: 97 pgs: 97 active+clean; 357 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 392 op/s 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: Reconfiguring grafana.smithi012 (dependencies changed)... 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:03.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:03 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T18:35:04.112 INFO:teuthology.orchestra.run.smithi012.stdout: 5 187682 9.87 MB/sec execute 355 sec latency 52.379 ms 2024-09-20T18:35:04.547 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:04 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T18:35:04.547 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:04 smithi012 ceph-mon[23745]: Reconfiguring daemon grafana.smithi012 on smithi012 2024-09-20T18:35:04.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:04 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-20T18:35:04.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:04 smithi174 ceph-mon[25682]: Reconfiguring daemon grafana.smithi012 on smithi012 2024-09-20T18:35:05.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 187926 9.86 MB/sec execute 356 sec latency 45.653 ms 2024-09-20T18:35:05.566 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:05 smithi012 ceph-mon[23745]: pgmap v19: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.6 MiB/s wr, 577 op/s 2024-09-20T18:35:05.769 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:05 smithi174 ceph-mon[25682]: pgmap v19: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.6 MiB/s wr, 577 op/s 2024-09-20T18:35:06.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 188280 9.87 MB/sec execute 357 sec latency 59.964 ms 2024-09-20T18:35:07.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 188596 9.87 MB/sec execute 358 sec latency 55.528 ms 2024-09-20T18:35:07.585 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:07 smithi012 ceph-mon[23745]: pgmap v20: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-09-20T18:35:07.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:07 smithi174 ceph-mon[25682]: pgmap v20: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-09-20T18:35:08.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 188840 9.86 MB/sec execute 359 sec latency 42.217 ms 2024-09-20T18:35:09.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 189126 9.85 MB/sec execute 360 sec latency 59.062 ms 2024-09-20T18:35:09.635 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:09 smithi012 ceph-mon[23745]: pgmap v21: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-09-20T18:35:09.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:09 smithi174 ceph-mon[25682]: pgmap v21: 97 pgs: 97 active+clean; 349 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 380 op/s 2024-09-20T18:35:10.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 189528 9.84 MB/sec execute 361 sec latency 29.397 ms 2024-09-20T18:35:11.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 189865 9.84 MB/sec execute 362 sec latency 43.382 ms 2024-09-20T18:35:11.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:10 smithi012 ceph-mon[23745]: pgmap v22: 97 pgs: 97 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 548 op/s 2024-09-20T18:35:11.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:10 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:11.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:10 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:11.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:10 smithi012 ceph-mon[23745]: Reconfiguring node-exporter.smithi012 (dependencies changed)... 2024-09-20T18:35:11.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:10 smithi012 ceph-mon[23745]: Deploying daemon node-exporter.smithi012 on smithi012 2024-09-20T18:35:11.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:10 smithi174 ceph-mon[25682]: pgmap v22: 97 pgs: 97 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 548 op/s 2024-09-20T18:35:11.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:10 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:11.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:10 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:11.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:10 smithi174 ceph-mon[25682]: Reconfiguring node-exporter.smithi012 (dependencies changed)... 2024-09-20T18:35:11.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:10 smithi174 ceph-mon[25682]: Deploying daemon node-exporter.smithi012 on smithi012 2024-09-20T18:35:12.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 190200 9.84 MB/sec execute 363 sec latency 45.716 ms 2024-09-20T18:35:13.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 190520 9.84 MB/sec execute 364 sec latency 60.255 ms 2024-09-20T18:35:13.243 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:12 smithi012 ceph-mon[23745]: pgmap v23: 97 pgs: 97 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 353 op/s 2024-09-20T18:35:13.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:12 smithi174 ceph-mon[25682]: pgmap v23: 97 pgs: 97 active+clean; 352 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 353 op/s 2024-09-20T18:35:14.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 190931 9.84 MB/sec execute 365 sec latency 55.775 ms 2024-09-20T18:35:14.624 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:35:15.027 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:15 smithi012 ceph-mon[23745]: pgmap v24: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-09-20T18:35:15.027 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:15 smithi012 ceph-mon[23745]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:15.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 191198 9.83 MB/sec execute 366 sec latency 43.564 ms 2024-09-20T18:35:15.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:15 smithi174 ceph-mon[25682]: pgmap v24: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.6 MiB/s wr, 566 op/s 2024-09-20T18:35:15.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:15 smithi174 ceph-mon[25682]: from='client.24671 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:15.640 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:35:15.640 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 starting - - - - 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (14m) 32s ago 14m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (14m) 37s ago 14m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 starting - - - - 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (8m) 32s ago 8m 39.7M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (8m) 37s ago 8m 14.3M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (15m) 32s ago 15m 176M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (64s) 37s ago 14m 495M - 19.3.0-5099-g88deadb8 441f9a3fe515 a5d2867ff390 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (16m) 32s ago 16m 57.2M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (14m) 37s ago 13m 45.8M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (8m) 32s ago 8m 254M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (8m) 37s ago 8m 83.4M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (14m) 32s ago 14m 21.3M - 1.3.1 1dbe0e931976 721d4388b70c 2024-09-20T18:35:15.641 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (13m) 37s ago 13m 20.3M - 1.3.1 1dbe0e931976 1c69bbee0c99 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (13m) 32s ago 13m 386M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (12m) 32s ago 12m 555M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (12m) 32s ago 12m 452M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (11m) 32s ago 11m 556M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (11m) 37s ago 11m 544M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (10m) 37s ago 10m 402M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (10m) 37s ago 10m 577M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (10m) 37s ago 10m 478M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:35:15.642 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (13m) 32s ago 13m 67.8M - 2.33.4 514e6a882f6e ef74fb47ac7d 2024-09-20T18:35:16.036 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:16 smithi012 ceph-mon[23745]: from='client.14966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:16.036 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:16 smithi012 ceph-mon[23745]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:16.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 191440 9.83 MB/sec execute 367 sec latency 42.667 ms 2024-09-20T18:35:16.175 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:35:16.175 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:16.176 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:35:16.177 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T18:35:16.177 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:35:16.177 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:35:16.177 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:35:16.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:16 smithi174 ceph-mon[25682]: from='client.14966 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:16.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:16 smithi174 ceph-mon[25682]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:16.653 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "", 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:35:16.654 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:35:17.067 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:17 smithi012 ceph-mon[23745]: pgmap v25: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-09-20T18:35:17.067 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:17 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2406617018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:35:17.067 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:17 smithi012 ceph-mon[23745]: from='client.14978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:17.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 191805 9.83 MB/sec execute 368 sec latency 56.226 ms 2024-09-20T18:35:17.270 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:35:17.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:17 smithi174 ceph-mon[25682]: pgmap v25: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-09-20T18:35:17.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:17 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2406617018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:35:17.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:17 smithi174 ceph-mon[25682]: from='client.14978 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:18.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 192070 9.82 MB/sec execute 369 sec latency 103.044 ms 2024-09-20T18:35:18.333 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:18 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/4150485393' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:35:18.333 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:18 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:18.333 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:18 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:18.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:18 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/4150485393' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:35:18.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:18 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:18.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:18 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:19.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 192336 9.82 MB/sec execute 370 sec latency 63.772 ms 2024-09-20T18:35:19.329 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:19 smithi012 ceph-mon[23745]: pgmap v26: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-09-20T18:35:19.329 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:19 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:19.329 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:19 smithi012 ceph-mon[23745]: Reconfiguring prometheus.smithi012 (dependencies changed)... 2024-09-20T18:35:19.329 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:19 smithi012 ceph-mon[23745]: Deploying daemon prometheus.smithi012 on smithi012 2024-09-20T18:35:19.431 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:19 smithi174 ceph-mon[25682]: pgmap v26: 97 pgs: 97 active+clean; 351 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 380 op/s 2024-09-20T18:35:19.432 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:19 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:19.432 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:19 smithi174 ceph-mon[25682]: Reconfiguring prometheus.smithi012 (dependencies changed)... 2024-09-20T18:35:19.432 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:19 smithi174 ceph-mon[25682]: Deploying daemon prometheus.smithi012 on smithi012 2024-09-20T18:35:20.113 INFO:teuthology.orchestra.run.smithi012.stdout: 5 192571 9.81 MB/sec execute 371 sec latency 50.757 ms 2024-09-20T18:35:21.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 192933 9.81 MB/sec execute 372 sec latency 40.952 ms 2024-09-20T18:35:21.138 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:21 smithi012 ceph-mon[23745]: pgmap v27: 97 pgs: 97 active+clean; 353 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.8 MiB/s wr, 561 op/s 2024-09-20T18:35:21.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:21 smithi174 ceph-mon[25682]: pgmap v27: 97 pgs: 97 active+clean; 353 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.8 MiB/s wr, 561 op/s 2024-09-20T18:35:22.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 193239 9.79 MB/sec execute 373 sec latency 211.628 ms 2024-09-20T18:35:23.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 193557 9.79 MB/sec execute 374 sec latency 42.420 ms 2024-09-20T18:35:23.338 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:23 smithi012 ceph-mon[23745]: pgmap v28: 97 pgs: 97 active+clean; 353 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-09-20T18:35:23.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:23 smithi174 ceph-mon[25682]: pgmap v28: 97 pgs: 97 active+clean; 353 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.9 MiB/s wr, 393 op/s 2024-09-20T18:35:24.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 193922 9.79 MB/sec execute 375 sec latency 61.525 ms 2024-09-20T18:35:25.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 194209 9.78 MB/sec execute 376 sec latency 63.765 ms 2024-09-20T18:35:25.381 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:25 smithi012 ceph-mon[23745]: pgmap v29: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 586 op/s 2024-09-20T18:35:25.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:25 smithi174 ceph-mon[25682]: pgmap v29: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 586 op/s 2024-09-20T18:35:26.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 194545 9.78 MB/sec execute 377 sec latency 116.295 ms 2024-09-20T18:35:27.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 194765 9.77 MB/sec execute 378 sec latency 128.775 ms 2024-09-20T18:35:27.169 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:27 smithi012 ceph-mon[23745]: pgmap v30: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-20T18:35:27.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:27 smithi174 ceph-mon[25682]: pgmap v30: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-20T18:35:28.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 194999 9.77 MB/sec execute 379 sec latency 75.083 ms 2024-09-20T18:35:29.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 195312 9.78 MB/sec execute 380 sec latency 41.411 ms 2024-09-20T18:35:29.197 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:29 smithi012 ceph-mon[23745]: pgmap v31: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-20T18:35:29.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:29 smithi174 ceph-mon[25682]: pgmap v31: 97 pgs: 97 active+clean; 351 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-20T18:35:30.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 195683 9.77 MB/sec execute 381 sec latency 71.643 ms 2024-09-20T18:35:31.114 INFO:teuthology.orchestra.run.smithi012.stdout: 5 195920 9.76 MB/sec execute 382 sec latency 56.898 ms 2024-09-20T18:35:31.259 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:31 smithi012 ceph-mon[23745]: pgmap v32: 97 pgs: 97 active+clean; 352 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-20T18:35:31.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:31 smithi174 ceph-mon[25682]: pgmap v32: 97 pgs: 97 active+clean; 352 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.6 MiB/s wr, 550 op/s 2024-09-20T18:35:32.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 196076 9.75 MB/sec execute 383 sec latency 76.635 ms 2024-09-20T18:35:33.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 196326 9.74 MB/sec execute 384 sec latency 64.463 ms 2024-09-20T18:35:33.312 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:33 smithi012 ceph-mon[23745]: pgmap v33: 97 pgs: 97 active+clean; 352 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 369 op/s 2024-09-20T18:35:33.312 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:33 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:33.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:33 smithi174 ceph-mon[25682]: pgmap v33: 97 pgs: 97 active+clean; 352 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 369 op/s 2024-09-20T18:35:33.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:33 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:34.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 196550 9.72 MB/sec execute 385 sec latency 63.995 ms 2024-09-20T18:35:35.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 196838 9.72 MB/sec execute 386 sec latency 55.112 ms 2024-09-20T18:35:35.352 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:35 smithi012 ceph-mon[23745]: pgmap v34: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-20T18:35:35.519 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:35 smithi174 ceph-mon[25682]: pgmap v34: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 527 op/s 2024-09-20T18:35:36.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 197044 9.71 MB/sec execute 387 sec latency 73.091 ms 2024-09-20T18:35:37.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 197298 9.69 MB/sec execute 388 sec latency 110.424 ms 2024-09-20T18:35:37.166 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:37 smithi012 ceph-mon[23745]: pgmap v35: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 334 op/s 2024-09-20T18:35:37.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:37 smithi174 ceph-mon[25682]: pgmap v35: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 334 op/s 2024-09-20T18:35:38.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 197583 9.70 MB/sec execute 389 sec latency 58.888 ms 2024-09-20T18:35:39.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 197902 9.68 MB/sec execute 390 sec latency 82.389 ms 2024-09-20T18:35:39.197 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:39 smithi012 ceph-mon[23745]: pgmap v36: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 334 op/s 2024-09-20T18:35:39.502 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:39 smithi174 ceph-mon[25682]: pgmap v36: 97 pgs: 97 active+clean; 357 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 334 op/s 2024-09-20T18:35:40.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 198252 9.69 MB/sec execute 391 sec latency 71.954 ms 2024-09-20T18:35:40.750 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:40 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:40.750 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:40 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:40.750 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:40 smithi012 ceph-mon[23745]: Reconfiguring node-exporter.smithi174 (dependencies changed)... 2024-09-20T18:35:40.751 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:40 smithi012 ceph-mon[23745]: Deploying daemon node-exporter.smithi174 on smithi174 2024-09-20T18:35:40.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:40 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:40.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:40 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:40.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:40 smithi174 ceph-mon[25682]: Reconfiguring node-exporter.smithi174 (dependencies changed)... 2024-09-20T18:35:40.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:40 smithi174 ceph-mon[25682]: Deploying daemon node-exporter.smithi174 on smithi174 2024-09-20T18:35:41.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 198503 9.68 MB/sec execute 392 sec latency 64.281 ms 2024-09-20T18:35:41.819 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:41 smithi012 ceph-mon[23745]: pgmap v37: 97 pgs: 97 active+clean; 353 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 497 op/s 2024-09-20T18:35:41.980 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:41 smithi174 ceph-mon[25682]: pgmap v37: 97 pgs: 97 active+clean; 353 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 497 op/s 2024-09-20T18:35:42.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 198962 9.70 MB/sec execute 393 sec latency 68.906 ms 2024-09-20T18:35:43.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 199380 9.70 MB/sec execute 394 sec latency 56.386 ms 2024-09-20T18:35:43.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:43 smithi012 ceph-mon[23745]: pgmap v38: 97 pgs: 97 active+clean; 353 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 321 op/s 2024-09-20T18:35:44.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:43 smithi174 ceph-mon[25682]: pgmap v38: 97 pgs: 97 active+clean; 353 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.1 MiB/s wr, 321 op/s 2024-09-20T18:35:44.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 199658 9.69 MB/sec execute 395 sec latency 44.933 ms 2024-09-20T18:35:45.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 200040 9.70 MB/sec execute 396 sec latency 58.228 ms 2024-09-20T18:35:45.934 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: pgmap v39: 97 pgs: 97 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 522 op/s 2024-09-20T18:35:45.934 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:35:45.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:45 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:46.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 200377 9.69 MB/sec execute 397 sec latency 46.126 ms 2024-09-20T18:35:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: pgmap v39: 97 pgs: 97 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.3 MiB/s wr, 522 op/s 2024-09-20T18:35:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:35:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-20T18:35:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi012.front.sepia.ceph.com:9093"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:35:46.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:45 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:46.942 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:35:46.942 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T18:35:46.942 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:35:46.942 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T18:35:46.942 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:46.943 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:35:46.943 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:35:46.943 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:46.943 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:46 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:47.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 200649 9.68 MB/sec execute 398 sec latency 44.689 ms 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi012.front.sepia.ceph.com:3000"}]: dispatch 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi012.front.sepia.ceph.com:9095"}]: dispatch 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:46 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:47.861 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:35:47.967 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:47 smithi012 ceph-mon[23745]: pgmap v40: 97 pgs: 97 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-09-20T18:35:47.967 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:47 smithi012 ceph-mon[23745]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:47.967 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:47 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:47.968 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:47 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:35:47.968 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:47 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:35:47.968 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:47 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:35:47.968 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:47 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:35:48.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 200952 9.68 MB/sec execute 399 sec latency 41.627 ms 2024-09-20T18:35:48.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:47 smithi174 ceph-mon[25682]: pgmap v40: 97 pgs: 97 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-09-20T18:35:48.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:47 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:35:48.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:47 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:48.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:47 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:35:48.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:47 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:35:48.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:47 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:35:48.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:47 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 starting - - - - 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (15m) 65s ago 15m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (14m) 70s ago 14m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 starting - - - - 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (9m) 65s ago 9m 39.7M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (9m) 70s ago 9m 14.3M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:9283 running (16m) 65s ago 16m 176M - 17.2.0 e1d6a67b021e 6964996b9b13 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (97s) 70s ago 14m 495M - 19.3.0-5099-g88deadb8 441f9a3fe515 a5d2867ff390 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (16m) 65s ago 16m 57.2M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (14m) 70s ago 14m 45.8M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:35:48.856 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (8m) 65s ago 8m 254M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (8m) 70s ago 8m 83.4M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 starting - - - - 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 starting - - - - 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (13m) 65s ago 13m 386M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (13m) 65s ago 13m 555M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (12m) 65s ago 12m 452M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (12m) 65s ago 12m 556M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (11m) 70s ago 11m 544M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (11m) 70s ago 11m 402M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (11m) 70s ago 11m 577M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (10m) 70s ago 10m 478M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:35:48.857 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 starting - - - - 2024-09-20T18:35:49.007 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:48 smithi012 ceph-mon[23745]: Upgrade: Updating mgr.smithi012.xkxxrd 2024-09-20T18:35:49.007 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:48 smithi012 ceph-mon[23745]: Deploying daemon mgr.smithi012.xkxxrd on smithi012 2024-09-20T18:35:49.007 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:48 smithi012 ceph-mon[23745]: from='client.14986 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:49.007 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:48 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:49.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 201219 9.67 MB/sec execute 400 sec latency 65.227 ms 2024-09-20T18:35:49.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:48 smithi174 ceph-mon[25682]: Upgrade: Updating mgr.smithi012.xkxxrd 2024-09-20T18:35:49.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:48 smithi174 ceph-mon[25682]: Deploying daemon mgr.smithi012.xkxxrd on smithi012 2024-09-20T18:35:49.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:48 smithi174 ceph-mon[25682]: from='client.14986 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:49.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:48 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:35:49.510 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:35:49.511 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:35:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:35:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-20T18:35:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:35:49.512 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:35:49.512 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:35:50.100 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:35:50.100 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:35:50.101 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:35:50.101 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:35:50.101 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:35:50.101 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "1/23 daemons upgraded", 2024-09-20T18:35:50.101 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading mgr daemons", 2024-09-20T18:35:50.101 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:35:50.101 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:35:50.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 201673 9.67 MB/sec execute 401 sec latency 52.099 ms 2024-09-20T18:35:50.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:49 smithi012 ceph-mon[23745]: pgmap v41: 97 pgs: 97 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-09-20T18:35:50.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:49 smithi012 ceph-mon[23745]: from='client.14990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:50.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:49 smithi012 ceph-mon[23745]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:50.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:49 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1924251471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:35:50.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:49 smithi174 ceph-mon[25682]: pgmap v41: 97 pgs: 97 active+clean; 356 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-09-20T18:35:50.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:49 smithi174 ceph-mon[25682]: from='client.14990 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:50.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:49 smithi174 ceph-mon[25682]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:50.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:49 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1924251471' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:35:50.759 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:35:50.900 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:50 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/460166946' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:35:51.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 201943 9.67 MB/sec execute 402 sec latency 54.115 ms 2024-09-20T18:35:51.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:50 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/460166946' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:35:51.909 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:51 smithi012 ceph-mon[23745]: pgmap v42: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 574 op/s 2024-09-20T18:35:51.909 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:51 smithi012 ceph-mon[23745]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:52.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 202389 9.69 MB/sec execute 403 sec latency 66.549 ms 2024-09-20T18:35:52.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:51 smithi174 ceph-mon[25682]: pgmap v42: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.6 MiB/s wr, 574 op/s 2024-09-20T18:35:52.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:51 smithi174 ceph-mon[25682]: from='client.24707 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:35:52.925 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:52 smithi012 ceph-mon[23745]: pgmap v43: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-20T18:35:53.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 202750 9.69 MB/sec execute 404 sec latency 68.155 ms 2024-09-20T18:35:53.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:52 smithi174 ceph-mon[25682]: pgmap v43: 97 pgs: 97 active+clean; 351 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-20T18:35:53.982 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:53 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:53.982 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:53 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:53.982 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:53 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:35:54.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 203004 9.68 MB/sec execute 405 sec latency 66.070 ms 2024-09-20T18:35:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:53 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:53 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:54.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:53 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:35:54.999 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:54 smithi012 ceph-mon[23745]: pgmap v44: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 616 op/s 2024-09-20T18:35:55.115 INFO:teuthology.orchestra.run.smithi012.stdout: 5 203211 9.67 MB/sec execute 406 sec latency 69.861 ms 2024-09-20T18:35:55.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:54 smithi174 ceph-mon[25682]: pgmap v44: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.4 MiB/s wr, 616 op/s 2024-09-20T18:35:56.116 INFO:teuthology.orchestra.run.smithi012.stdout: 5 203514 9.67 MB/sec execute 407 sec latency 66.588 ms 2024-09-20T18:35:57.048 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:57 smithi012 ceph-mon[23745]: pgmap v45: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-20T18:35:57.116 INFO:teuthology.orchestra.run.smithi012.stdout: 5 203859 9.66 MB/sec execute 408 sec latency 73.795 ms 2024-09-20T18:35:57.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:57 smithi174 ceph-mon[25682]: pgmap v45: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-20T18:35:58.116 INFO:teuthology.orchestra.run.smithi012.stdout: 5 204172 9.65 MB/sec execute 409 sec latency 53.041 ms 2024-09-20T18:35:58.332 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:58 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:58.332 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:58 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:58.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:58 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:58.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:58 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:35:59.116 INFO:teuthology.orchestra.run.smithi012.stdout: 5 204537 9.66 MB/sec execute 410 sec latency 70.659 ms 2024-09-20T18:35:59.375 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:35:59 smithi012 ceph-mon[23745]: pgmap v46: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 415 op/s 2024-09-20T18:35:59.630 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:35:59 smithi174 ceph-mon[25682]: pgmap v46: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 415 op/s 2024-09-20T18:36:00.116 INFO:teuthology.orchestra.run.smithi012.stdout: 5 204799 9.64 MB/sec execute 411 sec latency 74.633 ms 2024-09-20T18:36:00.380 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:00 smithi012 ceph-mon[23745]: Standby manager daemon smithi012.xkxxrd restarted 2024-09-20T18:36:00.380 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:00 smithi012 ceph-mon[23745]: Standby manager daemon smithi012.xkxxrd started 2024-09-20T18:36:00.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:00 smithi174 ceph-mon[25682]: Standby manager daemon smithi012.xkxxrd restarted 2024-09-20T18:36:00.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:00 smithi174 ceph-mon[25682]: Standby manager daemon smithi012.xkxxrd started 2024-09-20T18:36:01.119 INFO:teuthology.orchestra.run.smithi012.stdout: 5 205114 9.63 MB/sec execute 412 sec latency 58.829 ms 2024-09-20T18:36:01.402 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:01 smithi012 ceph-mon[23745]: pgmap v47: 97 pgs: 97 active+clean; 351 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 604 op/s 2024-09-20T18:36:01.402 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:01 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/crt"}]: dispatch 2024-09-20T18:36:01.402 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:01 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:36:01.402 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:01 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/key"}]: dispatch 2024-09-20T18:36:01.402 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:01 smithi012 ceph-mon[23745]: mgrmap e28: smithi174.nxkptu(active, since 89s), standbys: smithi012.xkxxrd 2024-09-20T18:36:01.402 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:01 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:36:01.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:01 smithi174 ceph-mon[25682]: pgmap v47: 97 pgs: 97 active+clean; 351 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 604 op/s 2024-09-20T18:36:01.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:01 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/crt"}]: dispatch 2024-09-20T18:36:01.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:01 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:36:01.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:01 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi012.xkxxrd/key"}]: dispatch 2024-09-20T18:36:01.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:01 smithi174 ceph-mon[25682]: mgrmap e28: smithi174.nxkptu(active, since 89s), standbys: smithi012.xkxxrd 2024-09-20T18:36:01.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:01 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:36:02.119 INFO:teuthology.orchestra.run.smithi012.stdout: 5 205348 9.63 MB/sec execute 413 sec latency 51.712 ms 2024-09-20T18:36:03.119 INFO:teuthology.orchestra.run.smithi012.stdout: 5 205573 9.63 MB/sec execute 414 sec latency 46.311 ms 2024-09-20T18:36:03.992 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:03 smithi012 ceph-mon[23745]: pgmap v48: 97 pgs: 97 active+clean; 351 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-20T18:36:03.992 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:03.992 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:03.992 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:03 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:03.992 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:03.992 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:03 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:04.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:03 smithi174 ceph-mon[25682]: pgmap v48: 97 pgs: 97 active+clean; 351 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-20T18:36:04.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:04.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:04.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:03 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:04.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:04.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:03 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:04.119 INFO:teuthology.orchestra.run.smithi012.stdout: 5 205864 9.64 MB/sec execute 415 sec latency 55.173 ms 2024-09-20T18:36:05.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 206154 9.63 MB/sec execute 416 sec latency 52.518 ms 2024-09-20T18:36:05.827 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:05 smithi012 ceph-mon[23745]: pgmap v49: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 567 op/s 2024-09-20T18:36:05.827 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:05 smithi012 ceph-mon[23745]: mgrmap e29: smithi174.nxkptu(active, since 93s), standbys: smithi012.xkxxrd 2024-09-20T18:36:06.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:05 smithi174 ceph-mon[25682]: pgmap v49: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 567 op/s 2024-09-20T18:36:06.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:05 smithi174 ceph-mon[25682]: mgrmap e29: smithi174.nxkptu(active, since 93s), standbys: smithi012.xkxxrd 2024-09-20T18:36:06.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 206450 9.63 MB/sec execute 417 sec latency 58.427 ms 2024-09-20T18:36:07.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 206658 9.61 MB/sec execute 418 sec latency 42.607 ms 2024-09-20T18:36:07.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:07 smithi012 ceph-mon[23745]: pgmap v50: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-09-20T18:36:08.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:07 smithi174 ceph-mon[25682]: pgmap v50: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-09-20T18:36:08.119 INFO:teuthology.orchestra.run.smithi012.stdout: 5 206907 9.61 MB/sec execute 419 sec latency 78.355 ms 2024-09-20T18:36:09.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 207280 9.60 MB/sec execute 420 sec latency 43.613 ms 2024-09-20T18:36:09.905 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:09 smithi012 ceph-mon[23745]: pgmap v51: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-09-20T18:36:10.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:09 smithi174 ceph-mon[25682]: pgmap v51: 97 pgs: 97 active+clean; 355 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-09-20T18:36:10.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 207657 9.60 MB/sec execute 421 sec latency 64.193 ms 2024-09-20T18:36:11.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 207992 9.60 MB/sec execute 422 sec latency 46.746 ms 2024-09-20T18:36:11.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: pgmap v52: 97 pgs: 97 active+clean; 359 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 524 op/s 2024-09-20T18:36:11.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:11.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: Failing over to other MGR 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr fail", "who": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr fail", "who": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:11.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:11 smithi012 ceph-mon[23745]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T18:36:11.765 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: pgmap v52: 97 pgs: 97 active+clean; 359 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.2 MiB/s wr, 524 op/s 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: Upgrade: Need to upgrade myself (mgr.smithi174.nxkptu) 2024-09-20T18:36:11.766 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: Failing over to other MGR 2024-09-20T18:36:11.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 172.21.15.174:0/4132072879' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr fail", "who": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:11.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "mgr fail", "who": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:11.767 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:11 smithi174 ceph-mon[25682]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T18:36:12.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 208347 9.60 MB/sec execute 423 sec latency 43.717 ms 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd='[{"prefix": "mgr fail", "who": "smithi174.nxkptu"}]': finished 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: mgrmap e30: smithi012.xkxxrd(active, starting, since 0.928122s) 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"}]: dispatch 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"}]: dispatch 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:36:12.962 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:36:12.963 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:12.964 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:36:12.964 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:12.964 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:12 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:36:13.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.24623 ' entity='mgr.smithi174.nxkptu' cmd='[{"prefix": "mgr fail", "who": "smithi174.nxkptu"}]': finished 2024-09-20T18:36:13.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: mgrmap e30: smithi012.xkxxrd(active, starting, since 0.928122s) 2024-09-20T18:36:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:36:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:36:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"}]: dispatch 2024-09-20T18:36:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"}]: dispatch 2024-09-20T18:36:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"}]: dispatch 2024-09-20T18:36:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-20T18:36:13.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-20T18:36:13.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-20T18:36:13.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-20T18:36:13.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-20T18:36:13.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-20T18:36:13.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-20T18:36:13.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-20T18:36:13.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-20T18:36:13.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-20T18:36:13.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-20T18:36:13.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:36:13.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:13.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:36:13.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:13.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:12 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"}]: dispatch 2024-09-20T18:36:13.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 208665 9.59 MB/sec execute 424 sec latency 58.339 ms 2024-09-20T18:36:13.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:13 smithi012 ceph-mon[23745]: mgrmap e31: smithi012.xkxxrd(active, since 1.94452s) 2024-09-20T18:36:13.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:13 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:36:13.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:13 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:36:13.971 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:13 smithi012 ceph-mon[23745]: pgmap v3: 97 pgs: 97 active+clean; 357 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:36:14.103 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:13 smithi174 ceph-mon[25682]: mgrmap e31: smithi012.xkxxrd(active, since 1.94452s) 2024-09-20T18:36:14.103 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:13 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:36:14.103 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:13 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"}]: dispatch 2024-09-20T18:36:14.103 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:13 smithi174 ceph-mon[25682]: pgmap v3: 97 pgs: 97 active+clean; 357 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:36:14.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 208929 9.59 MB/sec execute 425 sec latency 60.223 ms 2024-09-20T18:36:15.045 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:14 smithi012 ceph-mon[23745]: [20/Sep/2024:18:36:14] ENGINE Bus STARTING 2024-09-20T18:36:15.045 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:14 smithi012 ceph-mon[23745]: [20/Sep/2024:18:36:14] ENGINE Serving on http://172.21.15.12:8765 2024-09-20T18:36:15.045 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:14 smithi012 ceph-mon[23745]: [20/Sep/2024:18:36:14] ENGINE Serving on https://172.21.15.12:7150 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:14 smithi012 ceph-mon[23745]: [20/Sep/2024:18:36:14] ENGINE Bus STARTED 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:14 smithi012 ceph-mon[23745]: [20/Sep/2024:18:36:14] ENGINE Error in HTTPServer.serve 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Traceback (most recent call last): 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._connections.run(self.expiration_interval) 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._run(expiration_interval) 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: s = self.context.wrap_socket( 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T18:36:15.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: return self.sslsocket_class._create( 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self.do_handshake() 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._sslobj.do_handshake() 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:14 smithi174 ceph-mon[25682]: [20/Sep/2024:18:36:14] ENGINE Bus STARTING 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:14 smithi174 ceph-mon[25682]: [20/Sep/2024:18:36:14] ENGINE Serving on http://172.21.15.12:8765 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:14 smithi174 ceph-mon[25682]: [20/Sep/2024:18:36:14] ENGINE Serving on https://172.21.15.12:7150 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:14 smithi174 ceph-mon[25682]: [20/Sep/2024:18:36:14] ENGINE Bus STARTED 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:14 smithi174 ceph-mon[25682]: [20/Sep/2024:18:36:14] ENGINE Error in HTTPServer.serve 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Traceback (most recent call last): 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T18:36:15.047 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._connections.run(self.expiration_interval) 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._run(expiration_interval) 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: s = self.context.wrap_socket( 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: return self.sslsocket_class._create( 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self.do_handshake() 2024-09-20T18:36:15.048 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T18:36:15.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._sslobj.do_handshake() 2024-09-20T18:36:15.049 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T18:36:15.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 209211 9.59 MB/sec execute 426 sec latency 54.204 ms 2024-09-20T18:36:15.826 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:15 smithi012 ceph-mon[23745]: mgrmap e32: smithi012.xkxxrd(active, since 3s) 2024-09-20T18:36:15.849 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:15 smithi174 ceph-mon[25682]: mgrmap e32: smithi012.xkxxrd(active, since 3s) 2024-09-20T18:36:16.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 209555 9.60 MB/sec execute 427 sec latency 56.670 ms 2024-09-20T18:36:17.111 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:17 smithi012 ceph-mon[23745]: pgmap v5: 97 pgs: 97 active+clean; 357 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:36:17.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 209948 9.60 MB/sec execute 428 sec latency 47.079 ms 2024-09-20T18:36:17.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:17 smithi174 ceph-mon[25682]: pgmap v5: 97 pgs: 97 active+clean; 357 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:36:18.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 210160 9.59 MB/sec execute 429 sec latency 56.502 ms 2024-09-20T18:36:18.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:18 smithi012 ceph-mon[23745]: mgrmap e33: smithi012.xkxxrd(active, since 6s) 2024-09-20T18:36:18.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:18 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:18.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:18 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:18.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:18 smithi174 ceph-mon[25682]: mgrmap e33: smithi012.xkxxrd(active, since 6s) 2024-09-20T18:36:18.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:18 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:18.421 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:18 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:19.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 210551 9.59 MB/sec execute 430 sec latency 43.617 ms 2024-09-20T18:36:19.132 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: pgmap v6: 97 pgs: 97 active+clean; 357 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:36:19.132 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:19.132 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:19.132 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:36:19.132 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:36:19.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:36:19.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:36:19.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:19 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:36:19.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: pgmap v6: 97 pgs: 97 active+clean; 357 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:36:19.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:19.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:19.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:36:19.438 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:36:19.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:36:19.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:36:19.439 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:19 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/3501620463' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:36:20.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 210867 9.58 MB/sec execute 431 sec latency 54.651 ms 2024-09-20T18:36:20.147 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:20 smithi012 ceph-mon[23745]: mgrmap e34: smithi012.xkxxrd(active, since 8s), standbys: smithi174.nxkptu 2024-09-20T18:36:20.147 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:20 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi174.nxkptu", "id": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:20.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:20 smithi174 ceph-mon[25682]: mgrmap e34: smithi012.xkxxrd(active, since 8s), standbys: smithi174.nxkptu 2024-09-20T18:36:20.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:20 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr metadata", "who": "smithi174.nxkptu", "id": "smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:21.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 211155 9.58 MB/sec execute 432 sec latency 93.602 ms 2024-09-20T18:36:21.157 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:21 smithi012 ceph-mon[23745]: pgmap v7: 97 pgs: 97 active+clean; 359 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 3.6 MiB/s wr, 326 op/s 2024-09-20T18:36:21.269 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:36:21.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:21 smithi174 ceph-mon[25682]: pgmap v7: 97 pgs: 97 active+clean; 359 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 29 KiB/s rd, 3.6 MiB/s wr, 326 op/s 2024-09-20T18:36:22.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 211391 9.57 MB/sec execute 433 sec latency 58.579 ms 2024-09-20T18:36:22.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:22 smithi012 ceph-mon[23745]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:22.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:22 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:22.180 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:22 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (81s) 0s ago 15m 17.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (15m) 0s ago 15m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (15m) 5s ago 15m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (73s) 0s ago 15m 45.5M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (9m) 0s ago 9m 40.4M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (9m) 5s ago 9m 15.1M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (30s) 0s ago 17m 494M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283 running (2m) 5s ago 15m 153M - 19.3.0-5099-g88deadb8 441f9a3fe515 a5d2867ff390 2024-09-20T18:36:22.417 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (17m) 0s ago 17m 60.9M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (15m) 5s ago 15m 52.9M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (9m) 0s ago 9m 279M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (9m) 5s ago 9m 83.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (65s) 0s ago 15m 8488k - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (38s) 5s ago 14m 9.80M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (14m) 0s ago 14m 463M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (13m) 0s ago 13m 360M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (13m) 0s ago 13m 536M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (12m) 0s ago 12m 383M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (12m) 5s ago 12m 625M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (12m) 5s ago 12m 489M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (11m) 5s ago 11m 394M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (11m) 5s ago 11m 609M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:36:22.418 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (43s) 0s ago 14m 41.7M - 2.51.0 1d3b7f56885b bb49d6a7c8ee 2024-09-20T18:36:22.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:22 smithi174 ceph-mon[25682]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:22.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:22 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:22.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:22 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:23.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 211685 9.56 MB/sec execute 434 sec latency 46.851 ms 2024-09-20T18:36:23.144 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:36:23.144 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:36:23.144 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:36:23.144 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:23.144 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:36:23.145 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:36:23.214 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:23 smithi012 ceph-mon[23745]: from='client.15046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:23.214 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:23 smithi012 ceph-mon[23745]: pgmap v8: 97 pgs: 97 active+clean; 359 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 2.8 MiB/s wr, 253 op/s 2024-09-20T18:36:23.214 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:23 smithi012 ceph-mon[23745]: from='client.15050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:23.214 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:23 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:23.214 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:23 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:23.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:23 smithi174 ceph-mon[25682]: from='client.15046 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:23.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:23 smithi174 ceph-mon[25682]: pgmap v8: 97 pgs: 97 active+clean; 359 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 2.8 MiB/s wr, 253 op/s 2024-09-20T18:36:23.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:23 smithi174 ceph-mon[25682]: from='client.15050 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:23.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:23 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:23.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:23 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:23.732 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "2/23 daemons upgraded", 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "", 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:36:23.733 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:36:24.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 212040 9.56 MB/sec execute 435 sec latency 38.265 ms 2024-09-20T18:36:24.250 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:24 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/2173469560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:36:24.440 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:36:24.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:24 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/2173469560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:36:25.120 INFO:teuthology.orchestra.run.smithi012.stdout: 5 212370 9.55 MB/sec execute 436 sec latency 79.872 ms 2024-09-20T18:36:25.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:25 smithi012 ceph-mon[23745]: from='client.15058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:25.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:25 smithi012 ceph-mon[23745]: pgmap v9: 97 pgs: 97 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 426 op/s 2024-09-20T18:36:25.254 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:25 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/3618118855' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:36:25.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:25 smithi174 ceph-mon[25682]: from='client.15058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:25.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:25 smithi174 ceph-mon[25682]: pgmap v9: 97 pgs: 97 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 426 op/s 2024-09-20T18:36:25.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:25 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/3618118855' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:36:26.121 INFO:teuthology.orchestra.run.smithi012.stdout: 5 212614 9.55 MB/sec execute 437 sec latency 80.285 ms 2024-09-20T18:36:27.121 INFO:teuthology.orchestra.run.smithi012.stdout: 5 213032 9.57 MB/sec execute 438 sec latency 43.033 ms 2024-09-20T18:36:27.335 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:27 smithi012 ceph-mon[23745]: pgmap v10: 97 pgs: 97 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-09-20T18:36:27.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:27 smithi174 ceph-mon[25682]: pgmap v10: 97 pgs: 97 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-09-20T18:36:28.121 INFO:teuthology.orchestra.run.smithi012.stdout: 5 213419 9.57 MB/sec execute 439 sec latency 48.674 ms 2024-09-20T18:36:28.344 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:28 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:28.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:28 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:29.121 INFO:teuthology.orchestra.run.smithi012.stdout: 5 213679 9.57 MB/sec execute 440 sec latency 88.170 ms 2024-09-20T18:36:29.615 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: pgmap v11: 97 pgs: 97 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-09-20T18:36:29.615 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.615 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.615 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:29 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: pgmap v11: 97 pgs: 97 active+clean; 354 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 21 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:36:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:36:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:36:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:29 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:30.121 INFO:teuthology.orchestra.run.smithi012.stdout: 5 213973 9.56 MB/sec execute 441 sec latency 59.934 ms 2024-09-20T18:36:30.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:30 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:36:30.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:30 smithi174 ceph-mon[25682]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:36:30.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:30 smithi174 ceph-mon[25682]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:36:30.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:30 smithi174 ceph-mon[25682]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:36:30.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:30 smithi174 ceph-mon[25682]: Reconfiguring prometheus.smithi012 (dependencies changed)... 2024-09-20T18:36:30.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:30 smithi174 ceph-mon[25682]: Reconfiguring daemon prometheus.smithi012 on smithi012 2024-09-20T18:36:30.620 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:30 smithi012 ceph-mon[23745]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:36:30.620 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:30 smithi012 ceph-mon[23745]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:36:30.620 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:30 smithi012 ceph-mon[23745]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:36:30.620 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:30 smithi012 ceph-mon[23745]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:36:30.620 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:30 smithi012 ceph-mon[23745]: Reconfiguring prometheus.smithi012 (dependencies changed)... 2024-09-20T18:36:30.620 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:30 smithi012 ceph-mon[23745]: Reconfiguring daemon prometheus.smithi012 on smithi012 2024-09-20T18:36:31.121 INFO:teuthology.orchestra.run.smithi012.stdout: 5 214263 9.55 MB/sec execute 442 sec latency 58.927 ms 2024-09-20T18:36:31.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:31 smithi012 ceph-mon[23745]: pgmap v12: 97 pgs: 97 active+clean; 361 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-09-20T18:36:31.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:31 smithi174 ceph-mon[25682]: pgmap v12: 97 pgs: 97 active+clean; 361 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-09-20T18:36:32.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 214573 9.55 MB/sec execute 443 sec latency 60.617 ms 2024-09-20T18:36:33.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 215113 9.56 MB/sec execute 444 sec latency 35.199 ms 2024-09-20T18:36:33.463 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:33 smithi012 ceph-mon[23745]: pgmap v13: 97 pgs: 97 active+clean; 361 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 390 op/s 2024-09-20T18:36:33.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:33 smithi174 ceph-mon[25682]: pgmap v13: 97 pgs: 97 active+clean; 361 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 390 op/s 2024-09-20T18:36:34.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 215765 9.56 MB/sec execute 445 sec latency 25.098 ms 2024-09-20T18:36:35.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 216159 9.57 MB/sec execute 446 sec latency 38.165 ms 2024-09-20T18:36:35.484 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:35 smithi012 ceph-mon[23745]: pgmap v14: 97 pgs: 97 active+clean; 353 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 599 op/s 2024-09-20T18:36:35.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:35 smithi174 ceph-mon[25682]: pgmap v14: 97 pgs: 97 active+clean; 353 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.9 MiB/s wr, 599 op/s 2024-09-20T18:36:36.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 216557 9.59 MB/sec execute 447 sec latency 44.822 ms 2024-09-20T18:36:37.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 216890 9.58 MB/sec execute 448 sec latency 66.336 ms 2024-09-20T18:36:37.503 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:37 smithi012 ceph-mon[23745]: pgmap v15: 97 pgs: 97 active+clean; 353 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 400 op/s 2024-09-20T18:36:37.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:37 smithi174 ceph-mon[25682]: pgmap v15: 97 pgs: 97 active+clean; 353 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 400 op/s 2024-09-20T18:36:38.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 217171 9.58 MB/sec execute 449 sec latency 42.885 ms 2024-09-20T18:36:39.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:38 smithi012 ceph-mon[23745]: pgmap v16: 97 pgs: 97 active+clean; 353 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 400 op/s 2024-09-20T18:36:39.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:38 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:39.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:38 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:39.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:38 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:36:39.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:38 smithi012 ceph-mon[23745]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:36:39.058 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:38 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:39.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 217346 9.57 MB/sec execute 450 sec latency 79.022 ms 2024-09-20T18:36:39.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:38 smithi174 ceph-mon[25682]: pgmap v16: 97 pgs: 97 active+clean; 353 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.2 MiB/s wr, 400 op/s 2024-09-20T18:36:39.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:38 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:39.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:38 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:39.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:38 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:36:39.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:38 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-20T18:36:39.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:38 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:40.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 217660 9.57 MB/sec execute 451 sec latency 39.102 ms 2024-09-20T18:36:41.093 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:40 smithi012 ceph-mon[23745]: pgmap v17: 97 pgs: 97 active+clean; 363 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 646 op/s 2024-09-20T18:36:41.093 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:40 smithi012 ceph-mon[23745]: Upgrade: Updating mgr.smithi174.nxkptu 2024-09-20T18:36:41.093 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:40 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:36:41.093 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:40 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:36:41.093 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:40 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:41.093 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:40 smithi012 ceph-mon[23745]: Deploying daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:36:41.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 217991 9.56 MB/sec execute 452 sec latency 87.897 ms 2024-09-20T18:36:41.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:40 smithi174 ceph-mon[25682]: pgmap v17: 97 pgs: 97 active+clean; 363 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.1 KiB/s rd, 6.7 MiB/s wr, 646 op/s 2024-09-20T18:36:41.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:40 smithi174 ceph-mon[25682]: Upgrade: Updating mgr.smithi174.nxkptu 2024-09-20T18:36:41.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:40 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-20T18:36:41.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:40 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-20T18:36:41.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:40 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:41.191 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:40 smithi174 ceph-mon[25682]: Deploying daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:36:42.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 218354 9.56 MB/sec execute 453 sec latency 45.165 ms 2024-09-20T18:36:43.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 218684 9.56 MB/sec execute 454 sec latency 54.532 ms 2024-09-20T18:36:43.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:43 smithi012 ceph-mon[23745]: pgmap v18: 97 pgs: 97 active+clean; 363 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-20T18:36:43.133 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:43 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:43.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:43 smithi174 ceph-mon[25682]: pgmap v18: 97 pgs: 97 active+clean; 363 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-20T18:36:43.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:43 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:44.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 219006 9.55 MB/sec execute 455 sec latency 51.533 ms 2024-09-20T18:36:45.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 219326 9.54 MB/sec execute 456 sec latency 43.505 ms 2024-09-20T18:36:45.201 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:45 smithi012 ceph-mon[23745]: pgmap v19: 97 pgs: 97 active+clean; 355 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 651 op/s 2024-09-20T18:36:45.201 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:45 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:45.201 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:45 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:45.201 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:45 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:45.463 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:45 smithi174 ceph-mon[25682]: pgmap v19: 97 pgs: 97 active+clean; 355 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 651 op/s 2024-09-20T18:36:45.463 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:45 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:45.463 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:45 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:45.464 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:45 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:46.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 219463 9.53 MB/sec execute 457 sec latency 95.301 ms 2024-09-20T18:36:47.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 219653 9.53 MB/sec execute 458 sec latency 77.435 ms 2024-09-20T18:36:47.283 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:47 smithi012 ceph-mon[23745]: pgmap v20: 97 pgs: 97 active+clean; 355 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-09-20T18:36:47.444 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:47 smithi174 ceph-mon[25682]: pgmap v20: 97 pgs: 97 active+clean; 355 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-09-20T18:36:48.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 219893 9.53 MB/sec execute 459 sec latency 65.525 ms 2024-09-20T18:36:49.071 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:48 smithi012 ceph-mon[23745]: pgmap v21: 97 pgs: 97 active+clean; 355 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 518 op/s 2024-09-20T18:36:49.071 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:48 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:49.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:48 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:49.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:48 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:49.072 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:48 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:49.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 220090 9.52 MB/sec execute 460 sec latency 95.988 ms 2024-09-20T18:36:49.214 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:48 smithi174 ceph-mon[25682]: pgmap v21: 97 pgs: 97 active+clean; 355 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.2 MiB/s wr, 518 op/s 2024-09-20T18:36:49.214 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:48 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:49.214 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:48 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:49.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:48 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:49.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:48 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:50.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 220379 9.52 MB/sec execute 461 sec latency 62.906 ms 2024-09-20T18:36:51.127 INFO:teuthology.orchestra.run.smithi012.stdout: 5 220692 9.52 MB/sec execute 462 sec latency 126.019 ms 2024-09-20T18:36:51.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:50 smithi012 ceph-mon[23745]: pgmap v22: 97 pgs: 97 active+clean; 357 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 597 op/s 2024-09-20T18:36:51.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:51 smithi012 ceph-mon[23745]: Standby manager daemon smithi174.nxkptu restarted 2024-09-20T18:36:51.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:51 smithi012 ceph-mon[23745]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:36:51.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:51 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:36:51.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:51 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:36:51.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:51 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:36:51.152 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:51 smithi012 ceph-mon[23745]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:36:51.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:50 smithi174 ceph-mon[25682]: pgmap v22: 97 pgs: 97 active+clean; 357 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.0 MiB/s wr, 597 op/s 2024-09-20T18:36:51.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:50 smithi174 ceph-mon[25682]: Standby manager daemon smithi174.nxkptu restarted 2024-09-20T18:36:51.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:50 smithi174 ceph-mon[25682]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:36:51.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:50 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:36:51.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:50 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:36:51.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:50 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:36:51.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:50 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/2709281024' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:36:52.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 221075 9.52 MB/sec execute 463 sec latency 42.968 ms 2024-09-20T18:36:52.167 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:52 smithi012 ceph-mon[23745]: mgrmap e35: smithi012.xkxxrd(active, since 40s), standbys: smithi174.nxkptu 2024-09-20T18:36:52.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:52 smithi174 ceph-mon[25682]: mgrmap e35: smithi012.xkxxrd(active, since 40s), standbys: smithi174.nxkptu 2024-09-20T18:36:53.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 221533 9.52 MB/sec execute 464 sec latency 45.079 ms 2024-09-20T18:36:53.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: pgmap v23: 97 pgs: 97 active+clean; 357 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 350 op/s 2024-09-20T18:36:53.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:53.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:36:53.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:53.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:36:53.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:53 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi012.xkxxrd"}]: dispatch 2024-09-20T18:36:53.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: pgmap v23: 97 pgs: 97 active+clean; 357 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 350 op/s 2024-09-20T18:36:53.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:53.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-20T18:36:53.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-20T18:36:53.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:36:53.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:53.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:53 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi012.xkxxrd"}]: dispatch 2024-09-20T18:36:54.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 222046 9.52 MB/sec execute 465 sec latency 36.984 ms 2024-09-20T18:36:54.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:54 smithi012 ceph-mon[23745]: Upgrade: Setting container_image for all mgr 2024-09-20T18:36:54.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:54 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi012.xkxxrd"}]': finished 2024-09-20T18:36:54.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:54 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:54.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:54 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.nxkptu"}]': finished 2024-09-20T18:36:54.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:54 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T18:36:54.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:54 smithi174 ceph-mon[25682]: Upgrade: Setting container_image for all mgr 2024-09-20T18:36:54.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:54 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi012.xkxxrd"}]': finished 2024-09-20T18:36:54.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:54 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.nxkptu"}]: dispatch 2024-09-20T18:36:54.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:54 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi174.nxkptu"}]': finished 2024-09-20T18:36:54.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:54 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-20T18:36:55.092 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:36:55.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 222607 9.53 MB/sec execute 466 sec latency 65.712 ms 2024-09-20T18:36:55.443 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:55 smithi012 ceph-mon[23745]: pgmap v24: 97 pgs: 97 active+clean; 364 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 531 op/s 2024-09-20T18:36:55.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:55 smithi174 ceph-mon[25682]: pgmap v24: 97 pgs: 97 active+clean; 364 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.4 MiB/s wr, 531 op/s 2024-09-20T18:36:56.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 222911 9.52 MB/sec execute 467 sec latency 61.444 ms 2024-09-20T18:36:56.306 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:36:56.306 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (115s) 4s ago 16m 17.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (16m) 4s ago 16m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (15m) 8s ago 15m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (107s) 4s ago 16m 45.0M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (10m) 4s ago 10m 40.5M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (10m) 8s ago 10m 15.2M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (64s) 4s ago 17m 510M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (12s) 8s ago 15m 65.6M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (17m) 4s ago 17m 62.3M 2048M 17.2.0 e1d6a67b021e 15a06395140c 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (15m) 8s ago 15m 54.5M 2048M 17.2.0 e1d6a67b021e 54819c550bc5 2024-09-20T18:36:56.307 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (9m) 4s ago 9m 277M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (9m) 8s ago 9m 83.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (99s) 4s ago 16m 8270k - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (71s) 8s ago 15m 10.1M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (14m) 4s ago 14m 504M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (14m) 4s ago 14m 379M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (13m) 4s ago 13m 577M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (13m) 4s ago 13m 396M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (13m) 8s ago 13m 379M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (12m) 8s ago 12m 514M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (12m) 8s ago 12m 434M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (11m) 8s ago 11m 649M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:36:56.308 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (19s) 4s ago 15m 43.5M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:36:56.742 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:56 smithi012 ceph-mon[23745]: from='client.24765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:56.742 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:56 smithi012 ceph-mon[23745]: Upgrade: Updating mon.smithi012 2024-09-20T18:36:56.742 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:56 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:56.742 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:56 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T18:36:56.742 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:56 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T18:36:56.743 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:56 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:56.743 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:56 smithi012 ceph-mon[23745]: Deploying daemon mon.smithi012 on smithi012 2024-09-20T18:36:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:56 smithi174 ceph-mon[25682]: from='client.24765 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:56 smithi174 ceph-mon[25682]: Upgrade: Updating mon.smithi012 2024-09-20T18:36:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:56 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:36:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:56 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-20T18:36:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:56 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-20T18:36:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:56 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-20T18:36:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:56 smithi174 ceph-mon[25682]: Deploying daemon mon.smithi012 on smithi012 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:36:57.005 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:36:57.006 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:36:57.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 223182 9.53 MB/sec execute 468 sec latency 59.726 ms 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "2/23 daemons upgraded", 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T18:36:57.562 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:36:57.563 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:36:57.753 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:57 smithi012 ceph-mon[23745]: from='client.24769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:57.753 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:57 smithi012 ceph-mon[23745]: pgmap v25: 97 pgs: 97 active+clean; 364 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.5 MiB/s wr, 335 op/s 2024-09-20T18:36:57.753 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:57 smithi012 ceph-mon[23745]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:57.753 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:57 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/1840900062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:36:57.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:57 smithi174 ceph-mon[25682]: from='client.24769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:57.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:57 smithi174 ceph-mon[25682]: pgmap v25: 97 pgs: 97 active+clean; 364 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.5 MiB/s wr, 335 op/s 2024-09-20T18:36:57.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:57 smithi174 ceph-mon[25682]: from='client.24773 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:57.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:57 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/1840900062' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-20T18:36:58.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 223513 9.54 MB/sec execute 469 sec latency 46.128 ms 2024-09-20T18:36:58.257 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:36:58.260 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:58 smithi012 systemd[1]: Stopping Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:36:58.768 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:58 smithi012 ceph-mon[23745]: from='client.15084 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:58.768 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:58 smithi012 ceph-mon[23745]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:58.768 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:58 smithi012 ceph-mon[23745]: from='client.? 172.21.15.12:0/210530164' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:36:58.768 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:58 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[23741]: 2024-09-20T18:36:58.601+0000 7f740519c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi012 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:36:58.769 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:58 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[23741]: 2024-09-20T18:36:58.601+0000 7f740519c700 -1 mon.smithi012@0(leader) e2 *** Got Signal Terminated *** 2024-09-20T18:36:58.769 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:58 smithi012 podman[65183]: 2024-09-20 18:36:58.6435756 +0000 UTC m=+0.222528530 container died 15a06395140cc6effb9f83e38adb041262111cc9d89125ecd0aa3b4912940487 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , name=centos-stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.expose-services=, version=8, architecture=x86_64, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git) 2024-09-20T18:36:58.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:58 smithi174 ceph-mon[25682]: from='client.15084 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:36:58.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:58 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-20T18:36:58.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:36:58 smithi174 ceph-mon[25682]: from='client.? 172.21.15.12:0/210530164' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-20T18:36:59.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 223856 9.53 MB/sec execute 470 sec latency 45.172 ms 2024-09-20T18:36:59.358 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:59 smithi012 podman[65183]: 2024-09-20 18:36:59.284814098 +0000 UTC m=+0.863767027 container cleanup 15a06395140cc6effb9f83e38adb041262111cc9d89125ecd0aa3b4912940487 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-type=git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD) 2024-09-20T18:36:59.358 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:59 smithi012 bash[65183]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012 2024-09-20T18:36:59.609 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:59 smithi012 podman[65194]: 2024-09-20 18:36:59.582875705 +0000 UTC m=+0.935193686 container remove 15a06395140cc6effb9f83e38adb041262111cc9d89125ecd0aa3b4912940487 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, version=8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, maintainer=Guillaume Abrioux , release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, distribution-scope=public, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-20T18:37:00.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:59 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service: Deactivated successfully. 2024-09-20T18:37:00.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:59 smithi012 systemd[1]: Stopped Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:37:00.123 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:36:59 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service: Consumed 16.773s CPU time. 2024-09-20T18:37:00.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 224111 9.53 MB/sec execute 471 sec latency 57.996 ms 2024-09-20T18:37:00.380 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:00 smithi012 systemd[1]: Starting Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:37:00.652 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:00 smithi012 podman[65329]: 2024-09-20 18:37:00.342651793 +0000 UTC m=+0.020164904 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:37:00.903 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:00 smithi012 podman[65329]: 2024-09-20 18:37:00.720386434 +0000 UTC m=+0.397899527 container create 208d32fa88d140e55acff111bdee058c0bb0a3df5e0bf8070e358942ca5b20b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, org.label-schema.vendor=CentOS) 2024-09-20T18:37:01.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 224303 9.52 MB/sec execute 472 sec latency 98.002 ms 2024-09-20T18:37:01.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 podman[65329]: 2024-09-20 18:37:01.097462849 +0000 UTC m=+0.774975951 container init 208d32fa88d140e55acff111bdee058c0bb0a3df5e0bf8070e358942ca5b20b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-20T18:37:01.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 podman[65329]: 2024-09-20 18:37:01.109036064 +0000 UTC m=+0.786549171 container start 208d32fa88d140e55acff111bdee058c0bb0a3df5e0bf8070e358942ca5b20b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:37:01.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T18:37:01.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev), process ceph-mon, pid 2 2024-09-20T18:37:01.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: pidfile_write: ignore empty --pid-file 2024-09-20T18:37:01.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: load: jerasure load: lrc 2024-09-20T18:37:01.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: RocksDB version: 7.9.2 2024-09-20T18:37:01.406 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Git sha 0 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Compile date 2024-09-20 10:48:39 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: DB SUMMARY 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: DB Session ID: 08MWTUPYF9SG9VF2J1JV 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: CURRENT file: CURRENT 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1143 Bytes 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi012/store.db dir, Total Num: 1, files: 000033.sst 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi012/store.db: 000031.log size: 79808 ; 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.error_if_exists: 0 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.create_if_missing: 0 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.paranoid_checks: 1 2024-09-20T18:37:01.407 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.env: 0x55ce12718120 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.fs: PosixFileSystem 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.info_log: 0x55ce13f098e0 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.statistics: (nil) 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.use_fsync: 0 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_log_file_size: 0 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T18:37:01.408 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.allow_fallocate: 1 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.use_direct_reads: 0 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.db_log_dir: 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.wal_dir: 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T18:37:01.409 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.write_buffer_manager: 0x55ce13f0db80 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.unordered_write: 0 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T18:37:01.410 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.row_cache: None 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.wal_filter: None 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.two_write_queues: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.wal_compression: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.atomic_flush: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.log_readahead_size: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T18:37:01.411 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T18:37:01.412 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T18:37:01.412 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T18:37:01.412 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T18:37:01.412 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-20T18:37:01.412 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_background_jobs: 2 2024-09-20T18:37:01.412 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_background_compactions: -1 2024-09-20T18:37:01.412 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_subcompactions: 1 2024-09-20T18:37:01.413 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T18:37:01.413 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T18:37:01.413 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T18:37:01.413 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T18:37:01.413 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T18:37:01.413 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_open_files: -1 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_background_flushes: -1 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Compression algorithms supported: 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kZSTD supported: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kXpressCompression supported: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kBZip2Compression supported: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T18:37:01.414 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kLZ4Compression supported: 1 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kZlibCompression supported: 1 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: kSnappyCompression supported: 1 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi012/store.db/MANIFEST-000015 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.merge_operator: 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_filter: None 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T18:37:01.415 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ce13f09540) 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: cache_index_and_filter_blocks: 1 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: pin_top_level_index_and_filter: 1 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: index_type: 0 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: data_block_index_type: 0 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: index_shortening: 1 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: checksum: 4 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: no_block_cache: 0 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache: 0x55ce13efd610 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache_name: BinnedLRUCache 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache_options: 2024-09-20T18:37:01.416 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: capacity : 536870912 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: num_shard_bits : 4 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: strict_capacity_limit : 0 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: high_pri_pool_ratio: 0.000 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_cache_compressed: (nil) 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: persistent_cache: (nil) 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_size: 4096 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_size_deviation: 10 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_restart_interval: 16 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: index_block_restart_interval: 1 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: metadata_block_size: 4096 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: partition_filters: 0 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: use_delta_encoding: 1 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: filter_policy: bloomfilter 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: whole_key_filtering: 1 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: verify_compression: 0 2024-09-20T18:37:01.417 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: read_amp_bytes_per_bit: 0 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: format_version: 5 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: enable_index_compression: 1 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: block_align: 0 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: max_auto_readahead_size: 262144 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: prepopulate_block_cache: 0 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: initial_auto_readahead_size: 8192 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: num_file_reads_for_auto_readahead: 2 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression: NoCompression 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.num_levels: 7 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T18:37:01.418 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T18:37:01.419 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T18:37:01.419 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T18:37:01.419 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T18:37:01.419 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T18:37:01.419 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T18:37:01.422 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T18:37:01.422 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T18:37:01.422 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T18:37:01.422 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-20T18:37:01.422 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-20T18:37:01.422 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T18:37:01.423 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T18:37:01.424 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.arena_block_size: 1048576 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T18:37:01.425 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.inplace_update_support: 0 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.bloom_locality: 0 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.max_successive_merges: 0 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T18:37:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.ttl: 2592000 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.enable_blob_files: false 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.min_blob_size: 0 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-20T18:37:01.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.blob_file_starting_level: 0 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi012/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 16212, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6d3e43f4-b963-4a4d-8e9d-c094fdd17e42 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726857421141932, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726857421142772, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 79075, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 16206, "largest_seqno": 16351, "table_properties": {"data_size": 77442, "index_size": 339, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 261, "raw_key_size": 2455, "raw_average_key_size": 27, "raw_value_size": 75494, "raw_average_value_size": 857, "num_data_blocks": 15, "num_entries": 88, "num_filter_entries": 88, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726857421, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6d3e43f4-b963-4a4d-8e9d-c094fdd17e42", "db_session_id": "08MWTUPYF9SG9VF2J1JV", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726857421142869, "job": 1, "event": "recovery_finished"} 2024-09-20T18:37:01.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi012/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ce13f2ce00 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: DB pointer 0x55ce1402e000 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** DB Stats ** 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T18:37:01.429 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: L0 1/0 77.22 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: L6 1/0 9.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Sum 2/0 9.53 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 149.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** Compaction Stats [default] ** 2024-09-20T18:37:01.430 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 149.6 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T18:37:01.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Cumulative compaction: 0.00 GB write, 13.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Interval compaction: 0.00 GB write, 13.11 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Block cache BinnedLRUCache@0x55ce13efd610#2 capacity: 512.00 MB usage: 0.73 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.31 KB,5.96046e-05%) IndexBlock(1,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: starting mon.smithi012 rank 0 at public addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] at bind addrs [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi012 fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???) e2 preinit fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???).mds e6 new map 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???).mds e6 print_map 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: e6 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-20T18:37:01.432 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: legacy client fscid: 1 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Filesystem 'foofs' (1) 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: fs_name foofs 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: epoch 4 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: created 2024-09-20T18:26:31.040656+0000 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: modified 2024-09-20T18:26:36.875439+0000 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: tableserver 0 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: root 0 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: session_timeout 60 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: session_autoclose 300 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: max_file_size 1099511627776 2024-09-20T18:37:01.433 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: max_xattr_size 65536 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: required_client_features {} 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: last_failure 0 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: last_failure_osd_epoch 0 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: max_mds 1 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: in 0 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: up {0=14488} 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: failed 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: damaged 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: stopped 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: data_pools [3] 2024-09-20T18:37:01.434 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: metadata_pool 2 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: inline_data disabled 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: balancer 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: bal_rank_mask -1 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: standby_count_wanted 1 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: qdb_cluster leader: 0 members: 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: [mds.foofs.smithi012.jjsnmc{0:14488} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.12:6834/1898222382,v1:172.21.15.12:6835/1898222382] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Standby daemons: 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: [mds.foofs.smithi174.zwzgjx{-1:24291} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-20T18:37:01.435 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T18:37:01.436 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T18:37:01.436 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T18:37:01.436 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-20T18:37:01.436 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 bash[65329]: 208d32fa88d140e55acff111bdee058c0bb0a3df5e0bf8070e358942ca5b20b0 2024-09-20T18:37:01.436 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 systemd[1]: Started Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: pgmap v27: 97 pgs: 97 active+clean; 358 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 517 op/s 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012 calling monitor election 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mon.smithi012 is new leader, mons smithi012,smithi174 in quorum (ranks 0,1) 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: monmap epoch 2 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: last_changed 2024-09-20T18:21:16.095481+0000 2024-09-20T18:37:01.931 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: created 2024-09-20T18:19:07.084846+0000 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: min_mon_release 17 (quincy) 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: election_strategy: 1 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.smithi012 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mgrmap e35: smithi012.xkxxrd(active, since 50s), standbys: smithi174.nxkptu 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: overall HEALTH_OK 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: mgrmap e36: smithi012.xkxxrd(active, since 50s), standbys: smithi174.nxkptu 2024-09-20T18:37:01.932 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:01 smithi012 ceph-mon[65348]: from='mgr.15010 ' entity='' 2024-09-20T18:37:02.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 224484 9.51 MB/sec execute 473 sec latency 69.730 ms 2024-09-20T18:37:02.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi012"}]: dispatch 2024-09-20T18:37:02.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "mon metadata", "id": "smithi174"}]: dispatch 2024-09-20T18:37:02.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: from='mgr.15010 172.21.15.12:0/2313519657' entity='mgr.smithi012.xkxxrd' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:37:02.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: pgmap v27: 97 pgs: 97 active+clean; 358 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 517 op/s 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: mon.smithi012 calling monitor election 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: mon.smithi012 is new leader, mons smithi012,smithi174 in quorum (ranks 0,1) 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: monmap epoch 2 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: last_changed 2024-09-20T18:21:16.095481+0000 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: created 2024-09-20T18:19:07.084846+0000 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: min_mon_release 17 (quincy) 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: election_strategy: 1 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.smithi012 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: osdmap e72: 8 total, 8 up, 8 in 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: mgrmap e35: smithi012.xkxxrd(active, since 50s), standbys: smithi174.nxkptu 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: overall HEALTH_OK 2024-09-20T18:37:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: mgrmap e36: smithi012.xkxxrd(active, since 50s), standbys: smithi174.nxkptu 2024-09-20T18:37:02.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:01 smithi174 ceph-mon[25682]: from='mgr.15010 ' entity='' 2024-09-20T18:37:03.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 224659 9.50 MB/sec execute 474 sec latency 57.038 ms 2024-09-20T18:37:04.024 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:03 smithi012 ceph-mon[65348]: mgrmap e37: smithi012.xkxxrd(active, since 51s), standbys: smithi174.nxkptu 2024-09-20T18:37:04.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 224930 9.49 MB/sec execute 475 sec latency 118.318 ms 2024-09-20T18:37:04.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:03 smithi174 ceph-mon[25682]: mgrmap e37: smithi012.xkxxrd(active, since 51s), standbys: smithi174.nxkptu 2024-09-20T18:37:05.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 225177 9.49 MB/sec execute 476 sec latency 55.048 ms 2024-09-20T18:37:06.128 INFO:teuthology.orchestra.run.smithi012.stdout: 5 225423 9.48 MB/sec execute 477 sec latency 56.137 ms 2024-09-20T18:37:07.129 INFO:teuthology.orchestra.run.smithi012.stdout: 5 225635 9.47 MB/sec execute 478 sec latency 51.707 ms 2024-09-20T18:37:08.129 INFO:teuthology.orchestra.run.smithi012.stdout: 5 225943 9.47 MB/sec execute 479 sec latency 98.024 ms 2024-09-20T18:37:08.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:08 smithi012 ceph-mon[65348]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:37:08.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:08 smithi012 ceph-mon[65348]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:37:08.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:08 smithi012 ceph-mon[65348]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:37:08.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:08 smithi012 ceph-mon[65348]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:37:08.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:08 smithi012 ceph-mon[65348]: Standby manager daemon smithi174.nxkptu restarted 2024-09-20T18:37:08.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:08 smithi012 ceph-mon[65348]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:37:08.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:08 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/crt"}]: dispatch 2024-09-20T18:37:08.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:08 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-20T18:37:08.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:08 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi174.nxkptu/key"}]: dispatch 2024-09-20T18:37:08.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:08 smithi174 ceph-mon[25682]: from='mgr.? 172.21.15.174:0/1085914250' entity='mgr.smithi174.nxkptu' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-20T18:37:08.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:08 smithi174 ceph-mon[25682]: Standby manager daemon smithi174.nxkptu restarted 2024-09-20T18:37:08.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:08 smithi174 ceph-mon[25682]: Standby manager daemon smithi174.nxkptu started 2024-09-20T18:37:09.129 INFO:teuthology.orchestra.run.smithi012.stdout: 5 226206 9.46 MB/sec execute 480 sec latency 73.815 ms 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: mgrmap e38: smithi012.xkxxrd(active, since 57s), standbys: smithi174.nxkptu 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: Active manager daemon smithi012.xkxxrd restarted 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: Activating manager daemon smithi012.xkxxrd 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: mgrmap e39: smithi012.xkxxrd(active, starting, since 0.00710364s), standbys: smithi174.nxkptu 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi012"} : dispatch 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi174"} : dispatch 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"} : dispatch 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"} : dispatch 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"} : dispatch 2024-09-20T18:37:09.150 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr metadata", "who": "smithi174.nxkptu", "id": "smithi174.nxkptu"} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata"} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata"} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata"} : dispatch 2024-09-20T18:37:09.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:09 smithi012 ceph-mon[65348]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:37:09.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: mgrmap e38: smithi012.xkxxrd(active, since 57s), standbys: smithi174.nxkptu 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: Active manager daemon smithi012.xkxxrd restarted 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: Activating manager daemon smithi012.xkxxrd 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: mgrmap e39: smithi012.xkxxrd(active, starting, since 0.00710364s), standbys: smithi174.nxkptu 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi012"} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi174"} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr metadata", "who": "smithi012.xkxxrd", "id": "smithi012.xkxxrd"} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr metadata", "who": "smithi174.nxkptu", "id": "smithi174.nxkptu"} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T18:37:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata"} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata"} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata"} : dispatch 2024-09-20T18:37:09.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:09 smithi174 ceph-mon[25682]: Manager daemon smithi012.xkxxrd is now available 2024-09-20T18:37:10.129 INFO:teuthology.orchestra.run.smithi012.stdout: 5 226455 9.44 MB/sec execute 481 sec latency 67.662 ms 2024-09-20T18:37:10.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:37:10.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-20T18:37:10.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"} : dispatch 2024-09-20T18:37:10.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:10.189 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"} : dispatch 2024-09-20T18:37:10.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:10 smithi012 ceph-mon[65348]: mgrmap e40: smithi012.xkxxrd(active, since 1.01619s), standbys: smithi174.nxkptu 2024-09-20T18:37:10.190 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:10 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:37:10.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:10 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:37:10.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:10 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-20T18:37:10.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:10 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/mirror_snapshot_schedule"} : dispatch 2024-09-20T18:37:10.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:10 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:10.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:10 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi012.xkxxrd/trash_purge_schedule"} : dispatch 2024-09-20T18:37:10.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:10 smithi174 ceph-mon[25682]: mgrmap e40: smithi012.xkxxrd(active, since 1.01619s), standbys: smithi174.nxkptu 2024-09-20T18:37:10.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:10 smithi174 ceph-mon[25682]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-20T18:37:11.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 226618 9.44 MB/sec execute 482 sec latency 56.275 ms 2024-09-20T18:37:12.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:11 smithi174 ceph-mon[25682]: pgmap v3: 97 pgs: 97 active+clean; 354 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:37:12.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:11 smithi174 ceph-mon[25682]: pgmap v4: 97 pgs: 97 active+clean; 354 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:37:12.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:11 smithi174 ceph-mon[25682]: mgrmap e41: smithi012.xkxxrd(active, since 2s), standbys: smithi174.nxkptu 2024-09-20T18:37:12.065 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:11 smithi012 ceph-mon[65348]: pgmap v3: 97 pgs: 97 active+clean; 354 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:37:12.065 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:11 smithi012 ceph-mon[65348]: pgmap v4: 97 pgs: 97 active+clean; 354 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:37:12.065 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:11 smithi012 ceph-mon[65348]: mgrmap e41: smithi012.xkxxrd(active, since 2s), standbys: smithi174.nxkptu 2024-09-20T18:37:12.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 226785 9.43 MB/sec execute 483 sec latency 57.468 ms 2024-09-20T18:37:13.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:12 smithi012 ceph-mon[65348]: [20/Sep/2024:18:37:12] ENGINE Bus STARTING 2024-09-20T18:37:13.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:12 smithi012 ceph-mon[65348]: [20/Sep/2024:18:37:12] ENGINE Serving on https://172.21.15.12:7150 2024-09-20T18:37:13.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:12 smithi012 ceph-mon[65348]: [20/Sep/2024:18:37:12] ENGINE Error in HTTPServer.serve 2024-09-20T18:37:13.078 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: Traceback (most recent call last): 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._connections.run(self.expiration_interval) 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._run(expiration_interval) 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: s = self.context.wrap_socket( 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: return self.sslsocket_class._create( 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T18:37:13.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self.do_handshake() 2024-09-20T18:37:13.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T18:37:13.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: self._sslobj.do_handshake() 2024-09-20T18:37:13.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T18:37:13.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:12 smithi012 ceph-mon[65348]: [20/Sep/2024:18:37:12] ENGINE Serving on http://172.21.15.12:8765 2024-09-20T18:37:13.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:12 smithi012 ceph-mon[65348]: [20/Sep/2024:18:37:12] ENGINE Bus STARTED 2024-09-20T18:37:13.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:12 smithi012 ceph-mon[65348]: pgmap v5: 97 pgs: 97 active+clean; 354 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:37:13.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 227106 9.44 MB/sec execute 484 sec latency 63.805 ms 2024-09-20T18:37:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:12 smithi174 ceph-mon[25682]: [20/Sep/2024:18:37:12] ENGINE Bus STARTING 2024-09-20T18:37:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:12 smithi174 ceph-mon[25682]: [20/Sep/2024:18:37:12] ENGINE Serving on https://172.21.15.12:7150 2024-09-20T18:37:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:12 smithi174 ceph-mon[25682]: [20/Sep/2024:18:37:12] ENGINE Error in HTTPServer.serve 2024-09-20T18:37:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Traceback (most recent call last): 2024-09-20T18:37:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-20T18:37:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._connections.run(self.expiration_interval) 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._run(expiration_interval) 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: s = self.context.wrap_socket( 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: return self.sslsocket_class._create( 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self.do_handshake() 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: self._sslobj.do_handshake() 2024-09-20T18:37:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-20T18:37:13.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:12 smithi174 ceph-mon[25682]: [20/Sep/2024:18:37:12] ENGINE Serving on http://172.21.15.12:8765 2024-09-20T18:37:13.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:12 smithi174 ceph-mon[25682]: [20/Sep/2024:18:37:12] ENGINE Bus STARTED 2024-09-20T18:37:13.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:12 smithi174 ceph-mon[25682]: pgmap v5: 97 pgs: 97 active+clean; 354 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-20T18:37:14.101 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:13 smithi012 ceph-mon[65348]: mgrmap e42: smithi012.xkxxrd(active, since 4s), standbys: smithi174.nxkptu 2024-09-20T18:37:14.101 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:13 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:14.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 227450 9.44 MB/sec execute 485 sec latency 64.949 ms 2024-09-20T18:37:14.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:13 smithi174 ceph-mon[25682]: mgrmap e42: smithi012.xkxxrd(active, since 4s), standbys: smithi174.nxkptu 2024-09-20T18:37:14.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:13 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:15.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 227757 9.44 MB/sec execute 486 sec latency 51.490 ms 2024-09-20T18:37:15.143 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:14 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:15.143 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:14 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:15.143 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:14 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:15.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:14 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:15.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:14 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:15.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:14 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:16.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 227995 9.44 MB/sec execute 487 sec latency 52.710 ms 2024-09-20T18:37:16.692 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:16 smithi012 ceph-mon[65348]: pgmap v6: 97 pgs: 97 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 35 KiB/s rd, 2.5 MiB/s wr, 174 op/s 2024-09-20T18:37:17.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:16 smithi174 ceph-mon[25682]: pgmap v6: 97 pgs: 97 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 35 KiB/s rd, 2.5 MiB/s wr, 174 op/s 2024-09-20T18:37:17.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 228222 9.43 MB/sec execute 488 sec latency 51.782 ms 2024-09-20T18:37:18.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 228525 9.42 MB/sec execute 489 sec latency 44.499 ms 2024-09-20T18:37:18.762 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:18 smithi012 ceph-mon[65348]: pgmap v7: 97 pgs: 97 active+clean; 365 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 28 KiB/s rd, 3.3 MiB/s wr, 266 op/s 2024-09-20T18:37:18.762 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:18 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:18.762 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:18 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:19.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:18 smithi174 ceph-mon[25682]: pgmap v7: 97 pgs: 97 active+clean; 365 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 28 KiB/s rd, 3.3 MiB/s wr, 266 op/s 2024-09-20T18:37:19.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:18 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:19.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:18 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:19.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 228790 9.41 MB/sec execute 490 sec latency 58.360 ms 2024-09-20T18:37:19.778 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:19.778 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:19.778 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:19 smithi012 ceph-mon[65348]: pgmap v8: 97 pgs: 97 active+clean; 365 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 2.5 MiB/s wr, 207 op/s 2024-09-20T18:37:19.778 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:19.778 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:19.778 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:19.778 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:37:20.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:19 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:19 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:19 smithi174 ceph-mon[25682]: pgmap v8: 97 pgs: 97 active+clean; 365 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 2.5 MiB/s wr, 207 op/s 2024-09-20T18:37:20.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:19 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:19 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:19 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:20.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:19 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:37:20.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 228980 9.41 MB/sec execute 491 sec latency 88.965 ms 2024-09-20T18:37:20.819 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:37:20.819 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:20.820 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:21.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: Updating smithi012:/etc/ceph/ceph.conf 2024-09-20T18:37:21.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.conf 2024-09-20T18:37:21.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:37:21.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.conf 2024-09-20T18:37:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: Updating smithi174:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:37:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:20 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:21.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 229193 9.39 MB/sec execute 492 sec latency 66.270 ms 2024-09-20T18:37:21.853 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:21 smithi012 ceph-mon[65348]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:37:21.854 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:21 smithi012 ceph-mon[65348]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:37:21.854 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:21 smithi012 ceph-mon[65348]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:37:21.854 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:21 smithi012 ceph-mon[65348]: pgmap v9: 97 pgs: 97 active+clean; 356 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 3.8 MiB/s wr, 360 op/s 2024-09-20T18:37:21.854 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:21 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:37:21.854 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:21 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:37:21.854 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:21 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "quorum_status"} : dispatch 2024-09-20T18:37:22.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:21 smithi174 ceph-mon[25682]: Updating smithi012:/etc/ceph/ceph.client.admin.keyring 2024-09-20T18:37:22.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:21 smithi174 ceph-mon[25682]: Updating smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:37:22.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:21 smithi174 ceph-mon[25682]: Updating smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/config/ceph.client.admin.keyring 2024-09-20T18:37:22.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:21 smithi174 ceph-mon[25682]: pgmap v9: 97 pgs: 97 active+clean; 356 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 22 KiB/s rd, 3.8 MiB/s wr, 360 op/s 2024-09-20T18:37:22.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:21 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:37:22.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:21 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:37:22.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:21 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "quorum_status"} : dispatch 2024-09-20T18:37:22.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 229568 9.39 MB/sec execute 493 sec latency 50.359 ms 2024-09-20T18:37:23.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 229876 9.38 MB/sec execute 494 sec latency 61.518 ms 2024-09-20T18:37:23.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:23 smithi012 ceph-mon[65348]: Upgrade: Updating mon.smithi174 2024-09-20T18:37:23.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:23 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:23.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:23 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T18:37:23.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:23 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T18:37:23.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:23 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:23.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:23 smithi012 ceph-mon[65348]: Deploying daemon mon.smithi174 on smithi174 2024-09-20T18:37:23.630 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:23 smithi012 ceph-mon[65348]: pgmap v10: 97 pgs: 97 active+clean; 356 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 3.5 MiB/s wr, 329 op/s 2024-09-20T18:37:23.992 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:23 smithi174 ceph-mon[25682]: Upgrade: Updating mon.smithi174 2024-09-20T18:37:23.992 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:23 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:23.992 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:23 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T18:37:23.993 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:23 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T18:37:23.993 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:23 smithi174 ceph-mon[25682]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:23.993 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:23 smithi174 ceph-mon[25682]: Deploying daemon mon.smithi174 on smithi174 2024-09-20T18:37:23.993 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:23 smithi174 ceph-mon[25682]: pgmap v10: 97 pgs: 97 active+clean; 356 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 20 KiB/s rd, 3.5 MiB/s wr, 329 op/s 2024-09-20T18:37:24.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 230139 9.38 MB/sec execute 495 sec latency 67.683 ms 2024-09-20T18:37:24.615 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:24 smithi174 systemd[1]: Stopping Ceph mon.smithi174 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:37:25.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174[25663]: 2024-09-20T18:37:24.599+0000 7fa8b38ba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi174 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:37:25.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174[25663]: 2024-09-20T18:37:24.599+0000 7fa8b38ba700 -1 mon.smithi174@1(peon) e2 *** Got Signal Terminated *** 2024-09-20T18:37:25.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:24 smithi174 podman[49124]: 2024-09-20 18:37:24.762099441 +0000 UTC m=+0.291565973 container died 54819c550bc5262f804af0205fbf127fecfc9cced1bd4a1ed82c05c696b809cd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.expose-services=, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, release=754, RELEASE=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux ) 2024-09-20T18:37:25.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 230337 9.38 MB/sec execute 496 sec latency 57.188 ms 2024-09-20T18:37:25.466 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:25 smithi174 podman[49124]: 2024-09-20 18:37:25.183339745 +0000 UTC m=+0.712806270 container cleanup 54819c550bc5262f804af0205fbf127fecfc9cced1bd4a1ed82c05c696b809cd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, architecture=x86_64, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.expose-services=, RELEASE=HEAD, vcs-type=git, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_BRANCH=HEAD) 2024-09-20T18:37:25.466 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:25 smithi174 bash[49124]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174 2024-09-20T18:37:25.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:25 smithi174 podman[49138]: 2024-09-20 18:37:25.451727927 +0000 UTC m=+0.681980108 container remove 54819c550bc5262f804af0205fbf127fecfc9cced1bd4a1ed82c05c696b809cd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vcs-type=git, RELEASE=HEAD, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, ceph=True, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.expose-services=) 2024-09-20T18:37:25.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:25 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi174.service: Deactivated successfully. 2024-09-20T18:37:25.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:25 smithi174 systemd[1]: Stopped Ceph mon.smithi174 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:37:25.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:25 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi174.service: Consumed 14.314s CPU time. 2024-09-20T18:37:26.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 230701 9.39 MB/sec execute 497 sec latency 48.535 ms 2024-09-20T18:37:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:25 smithi174 systemd[1]: Starting Ceph mon.smithi174 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:37:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 podman[49260]: 2024-09-20 18:37:25.956421755 +0000 UTC m=+0.019061902 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:37:26.190 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 podman[49260]: 2024-09-20 18:37:26.06154245 +0000 UTC m=+0.124182606 container create 78d00d99c236bd6f04a534ecfb989be3c5e1b28296c415444fd0750e2e4ea3b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 podman[49260]: 2024-09-20 18:37:26.297930028 +0000 UTC m=+0.360570183 container init 78d00d99c236bd6f04a534ecfb989be3c5e1b28296c415444fd0750e2e4ea3b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 podman[49260]: 2024-09-20 18:37:26.301302888 +0000 UTC m=+0.363943035 container start 78d00d99c236bd6f04a534ecfb989be3c5e1b28296c415444fd0750e2e4ea3b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: set uid:gid to 167:167 (ceph:ceph) 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev), process ceph-mon, pid 2 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: pidfile_write: ignore empty --pid-file 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: load: jerasure load: lrc 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: RocksDB version: 7.9.2 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Git sha 0 2024-09-20T18:37:26.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Compile date 2024-09-20 10:48:39 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: DB SUMMARY 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: DB Session ID: BX36MBSSKF2WH8GH324N 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: CURRENT file: CURRENT 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: IDENTITY file: IDENTITY 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1071 Bytes 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi174/store.db dir, Total Num: 1, files: 000027.sst 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi174/store.db: 000025.log size: 3629375 ; 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.error_if_exists: 0 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.create_if_missing: 0 2024-09-20T18:37:26.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.paranoid_checks: 1 2024-09-20T18:37:26.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-20T18:37:26.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-20T18:37:26.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-20T18:37:26.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.env: 0x5577fcd4f120 2024-09-20T18:37:26.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.fs: PosixFileSystem 2024-09-20T18:37:26.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.info_log: 0x5577fed058e0 2024-09-20T18:37:26.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_file_opening_threads: 16 2024-09-20T18:37:26.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.statistics: (nil) 2024-09-20T18:37:26.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.use_fsync: 0 2024-09-20T18:37:26.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_log_file_size: 0 2024-09-20T18:37:26.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-20T18:37:26.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-20T18:37:26.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.keep_log_file_num: 1000 2024-09-20T18:37:26.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.recycle_log_file_num: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.allow_fallocate: 1 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.allow_mmap_reads: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.allow_mmap_writes: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.use_direct_reads: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.create_missing_column_families: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.db_log_dir: 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.wal_dir: 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-20T18:37:26.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.advise_random_on_open: 1 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.db_write_buffer_size: 0 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.write_buffer_manager: 0x5577fed09b80 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.rate_limiter: (nil) 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.wal_recovery_mode: 2 2024-09-20T18:37:26.776 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.enable_thread_tracking: 0 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.enable_pipelined_write: 0 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.unordered_write: 0 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.row_cache: None 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.wal_filter: None 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.allow_ingest_behind: 0 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.two_write_queues: 0 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.manual_wal_flush: 0 2024-09-20T18:37:26.777 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.wal_compression: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.atomic_flush: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.log_readahead_size: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.best_efforts_recovery: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.allow_data_in_errors: 0 2024-09-20T18:37:26.778 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.db_host_id: __hostname__ 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_background_jobs: 2 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_background_compactions: -1 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_subcompactions: 1 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_total_wal_size: 0 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-20T18:37:26.779 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_open_files: -1 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bytes_per_sync: 0 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_readahead_size: 0 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_background_flushes: -1 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Compression algorithms supported: 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kZSTD supported: 0 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kXpressCompression supported: 0 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kBZip2Compression supported: 0 2024-09-20T18:37:26.780 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kLZ4Compression supported: 1 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kZlibCompression supported: 1 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kLZ4HCCompression supported: 1 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: kSnappyCompression supported: 1 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000009 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.merge_operator: 2024-09-20T18:37:26.781 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_filter: None 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_filter_factory: None 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.sst_partitioner_factory: None 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5577fed05540) 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks: 1 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: pin_top_level_index_and_filter: 1 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_type: 0 2024-09-20T18:37:26.782 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_index_type: 0 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_shortening: 1 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: checksum: 4 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: no_block_cache: 0 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache: 0x5577fecf9610 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_name: BinnedLRUCache 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_options: 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: capacity : 536870912 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: num_shard_bits : 4 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: strict_capacity_limit : 0 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: high_pri_pool_ratio: 0.000 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_cache_compressed: (nil) 2024-09-20T18:37:26.783 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: persistent_cache: (nil) 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size: 4096 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_size_deviation: 10 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_restart_interval: 16 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: index_block_restart_interval: 1 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: metadata_block_size: 4096 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: partition_filters: 0 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: use_delta_encoding: 1 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: filter_policy: bloomfilter 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: whole_key_filtering: 1 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: verify_compression: 0 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: read_amp_bytes_per_bit: 0 2024-09-20T18:37:26.784 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: format_version: 5 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_index_compression: 1 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: block_align: 0 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_auto_readahead_size: 262144 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: prepopulate_block_cache: 0 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: initial_auto_readahead_size: 8192 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: num_file_reads_for_auto_readahead: 2 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.write_buffer_size: 33554432 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_write_buffer_number: 2 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression: NoCompression 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression: Disabled 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.prefix_extractor: nullptr 2024-09-20T18:37:26.785 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.num_levels: 7 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-20T18:37:26.786 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.level: 32767 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.strategy: 0 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.enabled: false 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-20T18:37:26.787 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.target_file_size_base: 67108864 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-20T18:37:26.788 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-20T18:37:26.789 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-20T18:37:26.789 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-20T18:37:26.789 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-20T18:37:26.789 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-20T18:37:26.789 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.arena_block_size: 1048576 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.disable_auto_compactions: 0 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-20T18:37:26.790 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.inplace_update_support: 0 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-20T18:37:26.791 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.bloom_locality: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.max_successive_merges: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.paranoid_file_checks: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.force_consistency_checks: 1 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.report_bg_io_stats: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.ttl: 2592000 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-20T18:37:26.792 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.enable_blob_files: false 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.min_blob_size: 0 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.blob_file_size: 268435456 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.blob_file_starting_level: 0 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T18:37:26.793 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi174/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 17133, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 43b43461-b1e4-41ba-bfd4-ac18da564158 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726857446330476, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726857446340500, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2335683, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17130, "largest_seqno": 17806, "table_properties": {"data_size": 2331754, "index_size": 2053, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 837, "raw_key_size": 8170, "raw_average_key_size": 25, "raw_value_size": 2324527, "raw_average_value_size": 7152, "num_data_blocks": 92, "num_entries": 325, "num_filter_entries": 325, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726857446, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "43b43461-b1e4-41ba-bfd4-ac18da564158", "db_session_id": "BX36MBSSKF2WH8GH324N", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726857446340623, "job": 1, "event": "recovery_finished"} 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi174/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5577fed28e00 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: DB pointer 0x5577fee2a000 2024-09-20T18:37:26.794 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** DB Stats ** 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-09-20T18:37:26.795 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L0 1/0 2.23 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 529.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: L6 1/0 9.45 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Sum 2/0 11.68 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 529.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 529.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** Compaction Stats [default] ** 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 529.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.796 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Cumulative compaction: 0.00 GB write, 176.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Interval compaction: 0.00 GB write, 176.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Block cache BinnedLRUCache@0x5577fecf9610#2 capacity: 512.00 MB usage: 3.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-20T18:37:26.797 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,2.33 KB,0.000444055%) Misc(1,0.00 KB,0%) 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: starting mon.smithi174 rank 1 at public addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] at bind addrs [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi174 fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???) e2 preinit fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???).mds e6 new map 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???).mds e6 print_map 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: e6 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T18:37:26.798 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: legacy client fscid: 1 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Filesystem 'foofs' (1) 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: fs_name foofs 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: epoch 4 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: created 2024-09-20T18:26:31.040656+0000 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: modified 2024-09-20T18:26:36.875439+0000 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: tableserver 0 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: root 0 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: session_timeout 60 2024-09-20T18:37:26.799 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: session_autoclose 300 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_file_size 1099511627776 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_xattr_size 65536 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: required_client_features {} 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: last_failure 0 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: last_failure_osd_epoch 0 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: max_mds 1 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: in 0 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: up {0=14488} 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: failed 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: damaged 2024-09-20T18:37:26.800 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: stopped 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: data_pools [3] 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: metadata_pool 2 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: inline_data disabled 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: balancer 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: bal_rank_mask -1 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: standby_count_wanted 1 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: qdb_cluster leader: 0 members: 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: [mds.foofs.smithi012.jjsnmc{0:14488} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.12:6834/1898222382,v1:172.21.15.12:6835/1898222382] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: Standby daemons: 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout: [mds.foofs.smithi174.zwzgjx{-1:24291} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] compat {c=[1],r=[1],i=[7ff]}] 2024-09-20T18:37:26.801 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-20T18:37:26.802 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T18:37:26.802 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T18:37:26.802 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-20T18:37:26.802 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 ceph-mon[49279]: mon.smithi174@-1(???).paxosservice(auth 1..31) refresh upgraded, format 0 -> 3 2024-09-20T18:37:26.802 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 bash[49260]: 78d00d99c236bd6f04a534ecfb989be3c5e1b28296c415444fd0750e2e4ea3b7 2024-09-20T18:37:26.802 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:26 smithi174 systemd[1]: Started Ceph mon.smithi174 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:37:27.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 231068 9.39 MB/sec execute 498 sec latency 53.811 ms 2024-09-20T18:37:27.724 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:27 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:28.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:27 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:28.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 231336 9.39 MB/sec execute 499 sec latency 41.160 ms 2024-09-20T18:37:28.784 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: pgmap v12: 97 pgs: 97 active+clean; 358 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 425 op/s 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi012"} : dispatch 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: mon.smithi012 calling monitor election 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: mon.smithi174 calling monitor election 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi174"} : dispatch 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: mon.smithi012 is new leader, mons smithi012,smithi174 in quorum (ranks 0,1) 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: monmap epoch 3 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: last_changed 2024-09-20T18:37:27.669896+0000 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: created 2024-09-20T18:19:07.084846+0000 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: min_mon_release 19 (squid) 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: election_strategy: 1 2024-09-20T18:37:28.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.smithi012 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: mgrmap e42: smithi012.xkxxrd(active, since 18s), standbys: smithi174.nxkptu 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: overall HEALTH_OK 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:28.786 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:37:28.847 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:37:29.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: pgmap v12: 97 pgs: 97 active+clean; 358 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 425 op/s 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi012"} : dispatch 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: mon.smithi012 calling monitor election 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: mon.smithi174 calling monitor election 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mon metadata", "id": "smithi174"} : dispatch 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: mon.smithi012 is new leader, mons smithi012,smithi174 in quorum (ranks 0,1) 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: monmap epoch 3 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: fsid c01c9b48-777c-11ef-baf5-efdc52797490 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: last_changed 2024-09-20T18:37:27.669896+0000 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: created 2024-09-20T18:19:07.084846+0000 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: min_mon_release 19 (squid) 2024-09-20T18:37:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: election_strategy: 1 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: 0: [v2:172.21.15.12:3300/0,v1:172.21.15.12:6789/0] mon.smithi012 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: 1: [v2:172.21.15.174:3300/0,v1:172.21.15.174:6789/0] mon.smithi174 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: osdmap e73: 8 total, 8 up, 8 in 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: mgrmap e42: smithi012.xkxxrd(active, since 18s), standbys: smithi174.nxkptu 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: overall HEALTH_OK 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:29.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:37:29.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 231504 9.38 MB/sec execute 500 sec latency 60.768 ms 2024-09-20T18:37:29.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:29 smithi012 ceph-mon[65348]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (2m) 11s ago 16m 17.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (16m) 11s ago 16m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (16m) 15s ago 16m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (2m) 11s ago 16m 45.6M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (10m) 11s ago 10m 40.5M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (10m) 15s ago 10m 15.4M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (97s) 11s ago 18m 493M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (46s) 15s ago 16m 452M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (29s) 11s ago 18m 30.5M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:37:29.815 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 starting - - - 2048M 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (10m) 11s ago 10m 286M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (10m) 15s ago 10m 83.6M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (2m) 11s ago 16m 8451k - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (105s) 15s ago 16m 10.0M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (15m) 11s ago 15m 521M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (14m) 11s ago 14m 404M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (14m) 11s ago 14m 324M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (13m) 11s ago 13m 411M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (13m) 15s ago 13m 410M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (13m) 15s ago 13m 531M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (12m) 15s ago 12m 461M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (12m) 15s ago 12m 680M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:37:29.816 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (52s) 11s ago 15m 43.9M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:37:29.940 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:29 smithi174 ceph-mon[49279]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:30.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 231786 9.38 MB/sec execute 501 sec latency 55.975 ms 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:37:30.392 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 4 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:37:30.393 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:37:30.834 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:30 smithi012 ceph-mon[65348]: pgmap v13: 97 pgs: 97 active+clean; 358 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.1 MiB/s wr, 342 op/s 2024-09-20T18:37:30.834 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:30 smithi012 ceph-mon[65348]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:30.834 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:30 smithi012 ceph-mon[65348]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:30.834 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:30 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2494787571' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:37:31.018 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:37:31.018 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:37:31.018 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:37:31.018 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:37:31.018 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:37:31.018 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:37:31.019 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:37:31.019 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "3/23 daemons upgraded", 2024-09-20T18:37:31.019 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T18:37:31.019 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:37:31.019 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:37:31.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:30 smithi174 ceph-mon[49279]: pgmap v13: 97 pgs: 97 active+clean; 358 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.1 MiB/s wr, 342 op/s 2024-09-20T18:37:31.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:30 smithi174 ceph-mon[49279]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:31.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:30 smithi174 ceph-mon[49279]: from='client.34138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:31.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:30 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2494787571' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:37:31.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 232078 9.36 MB/sec execute 502 sec latency 63.391 ms 2024-09-20T18:37:31.655 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:37:32.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 232382 9.36 MB/sec execute 503 sec latency 40.728 ms 2024-09-20T18:37:32.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:32 smithi012 ceph-mon[65348]: pgmap v14: 97 pgs: 97 active+clean; 365 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-09-20T18:37:32.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:32 smithi012 ceph-mon[65348]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:32.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:32.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:32.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:32 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2369731043' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:37:32.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:32.622 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:32.654 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:32 smithi174 ceph-mon[49279]: pgmap v14: 97 pgs: 97 active+clean; 365 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.2 MiB/s wr, 509 op/s 2024-09-20T18:37:32.654 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:32 smithi174 ceph-mon[49279]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:37:32.654 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:32.654 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:32.655 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:32 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2369731043' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:37:32.655 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:32.655 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:33.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 232627 9.36 MB/sec execute 504 sec latency 51.407 ms 2024-09-20T18:37:34.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 232994 9.36 MB/sec execute 505 sec latency 64.691 ms 2024-09-20T18:37:34.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:34 smithi012 ceph-mon[65348]: pgmap v15: 97 pgs: 97 active+clean; 365 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.6 MiB/s wr, 335 op/s 2024-09-20T18:37:34.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:34 smithi174 ceph-mon[49279]: pgmap v15: 97 pgs: 97 active+clean; 365 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.6 MiB/s wr, 335 op/s 2024-09-20T18:37:35.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 233338 9.35 MB/sec execute 506 sec latency 59.409 ms 2024-09-20T18:37:36.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 233623 9.35 MB/sec execute 507 sec latency 66.262 ms 2024-09-20T18:37:36.521 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: pgmap v16: 97 pgs: 97 active+clean; 362 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-09-20T18:37:36.521 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.521 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.521 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:36.521 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:37:36.521 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.522 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.522 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: Reconfiguring mon.smithi012 (monmap changed)... 2024-09-20T18:37:36.522 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T18:37:36.522 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T18:37:36.522 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:36.522 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:36 smithi012 ceph-mon[65348]: Reconfiguring daemon mon.smithi012 on smithi012 2024-09-20T18:37:36.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: pgmap v16: 97 pgs: 97 active+clean; 362 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 513 op/s 2024-09-20T18:37:36.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: Reconfiguring mon.smithi012 (monmap changed)... 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:36.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:36 smithi174 ceph-mon[49279]: Reconfiguring daemon mon.smithi012 on smithi012 2024-09-20T18:37:37.132 INFO:teuthology.orchestra.run.smithi012.stdout: 5 233868 9.34 MB/sec execute 508 sec latency 72.831 ms 2024-09-20T18:37:38.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 234190 9.35 MB/sec execute 509 sec latency 64.746 ms 2024-09-20T18:37:38.546 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: pgmap v17: 97 pgs: 97 active+clean; 361 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 372 op/s 2024-09-20T18:37:38.546 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:38.546 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:38.547 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: Reconfiguring mgr.smithi012.xkxxrd (monmap changed)... 2024-09-20T18:37:38.547 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T18:37:38.547 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T18:37:38.547 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:38.547 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:38 smithi012 ceph-mon[65348]: Reconfiguring daemon mgr.smithi012.xkxxrd on smithi012 2024-09-20T18:37:38.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: pgmap v17: 97 pgs: 97 active+clean; 361 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 372 op/s 2024-09-20T18:37:38.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: Reconfiguring mgr.smithi012.xkxxrd (monmap changed)... 2024-09-20T18:37:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi012.xkxxrd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T18:37:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T18:37:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:38 smithi174 ceph-mon[49279]: Reconfiguring daemon mgr.smithi012.xkxxrd on smithi012 2024-09-20T18:37:39.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 234537 9.35 MB/sec execute 510 sec latency 42.795 ms 2024-09-20T18:37:40.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 234761 9.34 MB/sec execute 511 sec latency 64.684 ms 2024-09-20T18:37:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: pgmap v18: 97 pgs: 97 active+clean; 361 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.6 MiB/s wr, 356 op/s 2024-09-20T18:37:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: Reconfiguring crash.smithi012 (monmap changed)... 2024-09-20T18:37:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:37:40.616 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:40.617 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: Reconfiguring daemon crash.smithi012 on smithi012 2024-09-20T18:37:40.617 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:41.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: pgmap v18: 97 pgs: 97 active+clean; 361 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.6 MiB/s wr, 356 op/s 2024-09-20T18:37:41.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:41.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:41.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: Reconfiguring crash.smithi012 (monmap changed)... 2024-09-20T18:37:41.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:37:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: Reconfiguring daemon crash.smithi012 on smithi012 2024-09-20T18:37:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:41.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 235014 9.34 MB/sec execute 512 sec latency 90.675 ms 2024-09-20T18:37:42.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 235251 9.34 MB/sec execute 513 sec latency 53.164 ms 2024-09-20T18:37:42.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:42 smithi012 ceph-mon[65348]: pgmap v19: 97 pgs: 97 active+clean; 366 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 532 op/s 2024-09-20T18:37:42.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:42.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:42.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T18:37:42.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:42 smithi174 ceph-mon[49279]: pgmap v19: 97 pgs: 97 active+clean; 366 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.6 MiB/s wr, 532 op/s 2024-09-20T18:37:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T18:37:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:43.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 235508 9.33 MB/sec execute 514 sec latency 50.371 ms 2024-09-20T18:37:43.840 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:43 smithi012 ceph-mon[65348]: Reconfiguring osd.0 (monmap changed)... 2024-09-20T18:37:43.840 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:43 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.0 on smithi012 2024-09-20T18:37:44.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:43 smithi174 ceph-mon[49279]: Reconfiguring osd.0 (monmap changed)... 2024-09-20T18:37:44.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:43 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.0 on smithi012 2024-09-20T18:37:44.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 235839 9.32 MB/sec execute 515 sec latency 57.064 ms 2024-09-20T18:37:44.597 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:44 smithi012 ceph-mon[65348]: pgmap v20: 97 pgs: 97 active+clean; 366 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-09-20T18:37:45.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:44 smithi174 ceph-mon[49279]: pgmap v20: 97 pgs: 97 active+clean; 366 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.6 MiB/s wr, 365 op/s 2024-09-20T18:37:45.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 236031 9.31 MB/sec execute 516 sec latency 54.357 ms 2024-09-20T18:37:45.861 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:45.861 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:45.861 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:45 smithi012 ceph-mon[65348]: Reconfiguring osd.1 (monmap changed)... 2024-09-20T18:37:45.861 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T18:37:45.862 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:45.862 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:45 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.1 on smithi012 2024-09-20T18:37:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:45 smithi174 ceph-mon[49279]: Reconfiguring osd.1 (monmap changed)... 2024-09-20T18:37:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T18:37:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:45 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.1 on smithi012 2024-09-20T18:37:46.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 236233 9.31 MB/sec execute 517 sec latency 77.349 ms 2024-09-20T18:37:46.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:46 smithi012 ceph-mon[65348]: pgmap v21: 97 pgs: 97 active+clean; 364 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.3 MiB/s wr, 406 op/s 2024-09-20T18:37:47.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:46 smithi174 ceph-mon[49279]: pgmap v21: 97 pgs: 97 active+clean; 364 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.3 MiB/s wr, 406 op/s 2024-09-20T18:37:47.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 236577 9.31 MB/sec execute 518 sec latency 61.336 ms 2024-09-20T18:37:48.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 236867 9.30 MB/sec execute 519 sec latency 72.141 ms 2024-09-20T18:37:48.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:48 smithi012 ceph-mon[65348]: pgmap v22: 97 pgs: 97 active+clean; 365 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 354 op/s 2024-09-20T18:37:48.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:48.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:48.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:48 smithi012 ceph-mon[65348]: Reconfiguring osd.2 (monmap changed)... 2024-09-20T18:37:48.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T18:37:48.179 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:48.180 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:48 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.2 on smithi012 2024-09-20T18:37:48.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:48 smithi174 ceph-mon[49279]: pgmap v22: 97 pgs: 97 active+clean; 365 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 354 op/s 2024-09-20T18:37:48.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:48.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:48.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:48 smithi174 ceph-mon[49279]: Reconfiguring osd.2 (monmap changed)... 2024-09-20T18:37:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T18:37:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:48 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.2 on smithi012 2024-09-20T18:37:49.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 237156 9.29 MB/sec execute 520 sec latency 65.951 ms 2024-09-20T18:37:50.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 237383 9.29 MB/sec execute 521 sec latency 61.475 ms 2024-09-20T18:37:50.738 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:50 smithi012 ceph-mon[65348]: pgmap v23: 97 pgs: 97 active+clean; 365 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 344 op/s 2024-09-20T18:37:50.738 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:50 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:50.738 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:50 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:50.739 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:50 smithi012 ceph-mon[65348]: Reconfiguring osd.3 (monmap changed)... 2024-09-20T18:37:50.739 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:50 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T18:37:50.739 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:50 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:50.739 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:50 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.3 on smithi012 2024-09-20T18:37:51.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:50 smithi174 ceph-mon[49279]: pgmap v23: 97 pgs: 97 active+clean; 365 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.6 MiB/s wr, 344 op/s 2024-09-20T18:37:51.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:50 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:51.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:50 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:51.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:50 smithi174 ceph-mon[49279]: Reconfiguring osd.3 (monmap changed)... 2024-09-20T18:37:51.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:50 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T18:37:51.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:50 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:51.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:50 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.3 on smithi012 2024-09-20T18:37:51.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 237588 9.29 MB/sec execute 522 sec latency 69.851 ms 2024-09-20T18:37:52.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 237858 9.29 MB/sec execute 523 sec latency 83.202 ms 2024-09-20T18:37:52.592 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:52 smithi174 ceph-mon[49279]: pgmap v24: 97 pgs: 97 active+clean; 358 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-09-20T18:37:52.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:52.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:52.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:52 smithi174 ceph-mon[49279]: Reconfiguring mds.foofs.smithi012.jjsnmc (monmap changed)... 2024-09-20T18:37:52.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:37:52.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:52.593 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:52 smithi174 ceph-mon[49279]: Reconfiguring daemon mds.foofs.smithi012.jjsnmc on smithi012 2024-09-20T18:37:52.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:52 smithi012 ceph-mon[65348]: pgmap v24: 97 pgs: 97 active+clean; 358 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.1 MiB/s wr, 510 op/s 2024-09-20T18:37:52.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:52.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:52.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:52 smithi012 ceph-mon[65348]: Reconfiguring mds.foofs.smithi012.jjsnmc (monmap changed)... 2024-09-20T18:37:52.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:37:52.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:52.781 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:52 smithi012 ceph-mon[65348]: Reconfiguring daemon mds.foofs.smithi012.jjsnmc on smithi012 2024-09-20T18:37:53.133 INFO:teuthology.orchestra.run.smithi012.stdout: 5 238155 9.28 MB/sec execute 524 sec latency 102.464 ms 2024-09-20T18:37:54.140 INFO:teuthology.orchestra.run.smithi012.stdout: 5 238365 9.29 MB/sec execute 525 sec latency 61.136 ms 2024-09-20T18:37:54.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: pgmap v25: 97 pgs: 97 active+clean; 358 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.1 MiB/s wr, 335 op/s 2024-09-20T18:37:54.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:54.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:54.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: Reconfiguring crash.smithi174 (monmap changed)... 2024-09-20T18:37:54.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:37:54.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:54.833 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: Reconfiguring daemon crash.smithi174 on smithi174 2024-09-20T18:37:54.833 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: pgmap v25: 97 pgs: 97 active+clean; 358 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.1 MiB/s wr, 335 op/s 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: Reconfiguring crash.smithi174 (monmap changed)... 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: Reconfiguring daemon crash.smithi174 on smithi174 2024-09-20T18:37:55.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:37:55.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 238562 9.27 MB/sec execute 526 sec latency 85.966 ms 2024-09-20T18:37:56.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 238782 9.27 MB/sec execute 527 sec latency 68.960 ms 2024-09-20T18:37:56.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: pgmap v26: 97 pgs: 97 active+clean; 362 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 446 op/s 2024-09-20T18:37:56.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:56.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:56.610 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: Reconfiguring mgr.smithi174.nxkptu (monmap changed)... 2024-09-20T18:37:56.611 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T18:37:56.611 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T18:37:56.611 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:56.611 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:56 smithi012 ceph-mon[65348]: Reconfiguring daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:37:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: pgmap v26: 97 pgs: 97 active+clean; 362 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 446 op/s 2024-09-20T18:37:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: Reconfiguring mgr.smithi174.nxkptu (monmap changed)... 2024-09-20T18:37:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi174.nxkptu", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-20T18:37:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mgr services"} : dispatch 2024-09-20T18:37:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:56.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:56 smithi174 ceph-mon[49279]: Reconfiguring daemon mgr.smithi174.nxkptu on smithi174 2024-09-20T18:37:57.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 239027 9.26 MB/sec execute 528 sec latency 75.089 ms 2024-09-20T18:37:58.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: pgmap v27: 97 pgs: 97 active+clean; 363 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 438 op/s 2024-09-20T18:37:58.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:58.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:58.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: Reconfiguring mon.smithi174 (monmap changed)... 2024-09-20T18:37:58.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T18:37:58.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T18:37:58.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:58.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:58 smithi012 ceph-mon[65348]: Reconfiguring daemon mon.smithi174 on smithi174 2024-09-20T18:37:58.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 239343 9.26 MB/sec execute 529 sec latency 63.869 ms 2024-09-20T18:37:58.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: pgmap v27: 97 pgs: 97 active+clean; 363 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 438 op/s 2024-09-20T18:37:58.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:58.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:58.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: Reconfiguring mon.smithi174 (monmap changed)... 2024-09-20T18:37:58.440 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-20T18:37:58.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-20T18:37:58.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:58.441 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:58 smithi174 ceph-mon[49279]: Reconfiguring daemon mon.smithi174 on smithi174 2024-09-20T18:37:59.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 239535 9.25 MB/sec execute 530 sec latency 64.448 ms 2024-09-20T18:37:59.660 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:59 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:59.660 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:59 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:59.660 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:59 smithi012 ceph-mon[65348]: Reconfiguring osd.4 (monmap changed)... 2024-09-20T18:37:59.660 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:59 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T18:37:59.660 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:59 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:59.660 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:59 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.4 on smithi174 2024-09-20T18:37:59.660 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:37:59 smithi012 ceph-mon[65348]: pgmap v28: 97 pgs: 97 active+clean; 363 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.1 MiB/s wr, 310 op/s 2024-09-20T18:37:59.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:59 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:59.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:59 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:37:59.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:59 smithi174 ceph-mon[49279]: Reconfiguring osd.4 (monmap changed)... 2024-09-20T18:37:59.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:59 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T18:37:59.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:59 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:37:59.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:59 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.4 on smithi174 2024-09-20T18:37:59.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:37:59 smithi174 ceph-mon[49279]: pgmap v28: 97 pgs: 97 active+clean; 363 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.1 MiB/s wr, 310 op/s 2024-09-20T18:38:00.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 239742 9.24 MB/sec execute 531 sec latency 70.227 ms 2024-09-20T18:38:01.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 240186 9.25 MB/sec execute 532 sec latency 43.837 ms 2024-09-20T18:38:01.203 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:01 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:01.203 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:01 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:01.203 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:01 smithi012 ceph-mon[65348]: Reconfiguring osd.5 (monmap changed)... 2024-09-20T18:38:01.203 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:01 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T18:38:01.203 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:01 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:01.204 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:01 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.5 on smithi174 2024-09-20T18:38:01.204 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:01 smithi012 ceph-mon[65348]: pgmap v29: 97 pgs: 97 active+clean; 355 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.6 MiB/s wr, 469 op/s 2024-09-20T18:38:01.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:01 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:01.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:01 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:01.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:01 smithi174 ceph-mon[49279]: Reconfiguring osd.5 (monmap changed)... 2024-09-20T18:38:01.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:01 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T18:38:01.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:01 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:01.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:01 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.5 on smithi174 2024-09-20T18:38:01.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:01 smithi174 ceph-mon[49279]: pgmap v29: 97 pgs: 97 active+clean; 355 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.6 MiB/s wr, 469 op/s 2024-09-20T18:38:02.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 240528 9.24 MB/sec execute 533 sec latency 74.986 ms 2024-09-20T18:38:02.249 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:38:02.744 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:02.744 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:02.744 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:02 smithi012 ceph-mon[65348]: Reconfiguring osd.6 (monmap changed)... 2024-09-20T18:38:02.744 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T18:38:02.745 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:02.745 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:02 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.6 on smithi174 2024-09-20T18:38:03.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:03.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:03.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:02 smithi174 ceph-mon[49279]: Reconfiguring osd.6 (monmap changed)... 2024-09-20T18:38:03.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T18:38:03.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:03.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:02 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.6 on smithi174 2024-09-20T18:38:03.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 240796 9.24 MB/sec execute 534 sec latency 69.950 ms 2024-09-20T18:38:03.335 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (3m) 44s ago 17m 17.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (17m) 44s ago 17m 6395k - 17.2.0 e1d6a67b021e 10b415afd641 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (16m) 31s ago 16m 6400k - 17.2.0 e1d6a67b021e bba2b70de89e 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (2m) 44s ago 17m 45.6M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (11m) 44s ago 11m 40.5M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (11m) 31s ago 11m 15.8M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (2m) 44s ago 18m 493M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (79s) 31s ago 16m 452M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (62s) 44s ago 18m 30.5M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (37s) 31s ago 16m 21.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (11m) 44s ago 11m 286M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (11m) 31s ago 11m 83.8M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (2m) 44s ago 17m 8451k - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:38:03.336 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (2m) 31s ago 16m 10.1M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (15m) 44s ago 15m 521M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (15m) 44s ago 15m 404M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (14m) 44s ago 14m 324M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (14m) 44s ago 14m 411M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (14m) 31s ago 14m 426M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (13m) 31s ago 13m 541M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (13m) 31s ago 13m 477M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (12m) 31s ago 12m 689M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:38:03.337 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (86s) 44s ago 16m 43.9M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:38:03.755 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:03 smithi012 ceph-mon[65348]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:03.755 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:03 smithi012 ceph-mon[65348]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:03.755 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:03 smithi012 ceph-mon[65348]: pgmap v30: 97 pgs: 97 active+clean; 355 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.2 MiB/s wr, 302 op/s 2024-09-20T18:38:03.755 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:03.755 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:03.756 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T18:38:03.756 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:03.844 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:03 smithi174 ceph-mon[49279]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:03.844 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:03 smithi174 ceph-mon[49279]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:03.845 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:03 smithi174 ceph-mon[49279]: pgmap v30: 97 pgs: 97 active+clean; 355 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.2 MiB/s wr, 302 op/s 2024-09-20T18:38:03.845 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:03.845 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:03.845 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T18:38:03.845 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:03.999 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:38:04.000 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:04.001 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:38:04.001 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T18:38:04.001 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 4 2024-09-20T18:38:04.001 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:38:04.001 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:38:04.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 241033 9.24 MB/sec execute 535 sec latency 53.377 ms 2024-09-20T18:38:04.542 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:38:04.542 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:38:04.542 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:38:04.542 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:38:04.542 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:38:04.542 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:38:04.543 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:38:04.543 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:38:04.543 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "4/23 daemons upgraded", 2024-09-20T18:38:04.543 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading mon daemons", 2024-09-20T18:38:04.543 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:38:04.543 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:38:04.802 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:04 smithi012 ceph-mon[65348]: Reconfiguring osd.7 (monmap changed)... 2024-09-20T18:38:04.802 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:04 smithi012 ceph-mon[65348]: Reconfiguring daemon osd.7 on smithi174 2024-09-20T18:38:04.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:04 smithi012 ceph-mon[65348]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:04.803 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:04 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/203295025' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:05.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:04 smithi174 ceph-mon[49279]: Reconfiguring osd.7 (monmap changed)... 2024-09-20T18:38:05.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:04 smithi174 ceph-mon[49279]: Reconfiguring daemon osd.7 on smithi174 2024-09-20T18:38:05.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:04 smithi174 ceph-mon[49279]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:05.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:04 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/203295025' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:05.146 INFO:teuthology.orchestra.run.smithi012.stdout: 5 241379 9.24 MB/sec execute 536 sec latency 59.774 ms 2024-09-20T18:38:05.233 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:38:05.825 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:05.825 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:05.825 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:05.825 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: Reconfiguring mds.foofs.smithi174.zwzgjx (monmap changed)... 2024-09-20T18:38:05.825 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:38:05.826 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:05.826 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: Reconfiguring daemon mds.foofs.smithi174.zwzgjx on smithi174 2024-09-20T18:38:05.826 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: pgmap v31: 97 pgs: 97 active+clean; 362 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 464 op/s 2024-09-20T18:38:05.826 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:05 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/1216766034' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:38:06.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: Reconfiguring mds.foofs.smithi174.zwzgjx (monmap changed)... 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: Reconfiguring daemon mds.foofs.smithi174.zwzgjx on smithi174 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: pgmap v31: 97 pgs: 97 active+clean; 362 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.5 MiB/s wr, 464 op/s 2024-09-20T18:38:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:05 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/1216766034' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:38:06.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 241670 9.24 MB/sec execute 537 sec latency 56.985 ms 2024-09-20T18:38:07.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 241879 9.23 MB/sec execute 538 sec latency 105.800 ms 2024-09-20T18:38:07.381 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all mon 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi012"} : dispatch 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi012"}]': finished 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"} : dispatch 2024-09-20T18:38:07.382 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:07 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]': finished 2024-09-20T18:38:07.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all mon 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi012"} : dispatch 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi012"}]': finished 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"} : dispatch 2024-09-20T18:38:07.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:07 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi174"}]': finished 2024-09-20T18:38:08.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 242124 9.23 MB/sec execute 539 sec latency 66.438 ms 2024-09-20T18:38:08.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:08 smithi012 ceph-mon[65348]: pgmap v32: 97 pgs: 97 active+clean; 359 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-09-20T18:38:09.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 242319 9.22 MB/sec execute 540 sec latency 44.833 ms 2024-09-20T18:38:09.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:08 smithi174 ceph-mon[49279]: pgmap v32: 97 pgs: 97 active+clean; 359 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-09-20T18:38:10.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 242546 9.22 MB/sec execute 541 sec latency 86.178 ms 2024-09-20T18:38:10.446 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:10 smithi012 ceph-mon[65348]: pgmap v33: 97 pgs: 97 active+clean; 359 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 342 op/s 2024-09-20T18:38:10.446 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:10 smithi012 ceph-mon[65348]: Upgrade: Updating crash.smithi012 (1/2) 2024-09-20T18:38:10.446 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:10.446 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:38:10.446 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:10.447 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:10 smithi012 ceph-mon[65348]: Deploying daemon crash.smithi012 on smithi012 2024-09-20T18:38:10.447 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:10.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:10 smithi174 ceph-mon[49279]: pgmap v33: 97 pgs: 97 active+clean; 359 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 342 op/s 2024-09-20T18:38:10.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:10 smithi174 ceph-mon[49279]: Upgrade: Updating crash.smithi012 (1/2) 2024-09-20T18:38:10.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:10.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi012", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:38:10.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:10.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:10 smithi174 ceph-mon[49279]: Deploying daemon crash.smithi012 on smithi012 2024-09-20T18:38:10.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:11.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 242720 9.21 MB/sec execute 542 sec latency 129.191 ms 2024-09-20T18:38:11.507 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:11 smithi012 ceph-mon[65348]: pgmap v34: 97 pgs: 97 active+clean; 364 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 492 op/s 2024-09-20T18:38:11.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:11 smithi174 ceph-mon[49279]: pgmap v34: 97 pgs: 97 active+clean; 364 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.1 MiB/s wr, 492 op/s 2024-09-20T18:38:12.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 242968 9.20 MB/sec execute 543 sec latency 104.112 ms 2024-09-20T18:38:13.046 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:12 smithi012 ceph-mon[65348]: pgmap v35: 97 pgs: 97 active+clean; 364 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.5 MiB/s wr, 334 op/s 2024-09-20T18:38:13.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 243192 9.19 MB/sec execute 544 sec latency 64.792 ms 2024-09-20T18:38:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:12 smithi174 ceph-mon[49279]: pgmap v35: 97 pgs: 97 active+clean; 364 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.5 MiB/s wr, 334 op/s 2024-09-20T18:38:14.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 243419 9.18 MB/sec execute 545 sec latency 64.854 ms 2024-09-20T18:38:15.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 243766 9.18 MB/sec execute 546 sec latency 72.674 ms 2024-09-20T18:38:15.873 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:15 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:15.873 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:15 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:15.873 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:15 smithi012 ceph-mon[65348]: pgmap v36: 97 pgs: 97 active+clean; 363 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 456 op/s 2024-09-20T18:38:16.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:15 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:16.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:15 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:16.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:15 smithi174 ceph-mon[49279]: pgmap v36: 97 pgs: 97 active+clean; 363 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.3 MiB/s wr, 456 op/s 2024-09-20T18:38:16.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 244081 9.17 MB/sec execute 547 sec latency 93.211 ms 2024-09-20T18:38:17.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 244374 9.17 MB/sec execute 548 sec latency 93.306 ms 2024-09-20T18:38:17.906 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:17 smithi012 ceph-mon[65348]: Upgrade: Updating crash.smithi174 (2/2) 2024-09-20T18:38:17.906 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:17.907 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:38:17.907 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:17.907 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:17 smithi012 ceph-mon[65348]: Deploying daemon crash.smithi174 on smithi174 2024-09-20T18:38:17.907 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:17 smithi012 ceph-mon[65348]: pgmap v37: 97 pgs: 97 active+clean; 354 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 326 op/s 2024-09-20T18:38:18.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 244619 9.18 MB/sec execute 549 sec latency 48.138 ms 2024-09-20T18:38:18.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:17 smithi174 ceph-mon[49279]: Upgrade: Updating crash.smithi174 (2/2) 2024-09-20T18:38:18.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:18.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi174", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-20T18:38:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:17 smithi174 ceph-mon[49279]: Deploying daemon crash.smithi174 on smithi174 2024-09-20T18:38:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:17 smithi174 ceph-mon[49279]: pgmap v37: 97 pgs: 97 active+clean; 354 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 326 op/s 2024-09-20T18:38:19.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:18 smithi174 ceph-mon[49279]: pgmap v38: 97 pgs: 97 active+clean; 354 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.0 MiB/s wr, 305 op/s 2024-09-20T18:38:19.147 INFO:teuthology.orchestra.run.smithi012.stdout: 5 244959 9.18 MB/sec execute 550 sec latency 59.680 ms 2024-09-20T18:38:19.212 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:18 smithi012 ceph-mon[65348]: pgmap v38: 97 pgs: 97 active+clean; 354 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.0 MiB/s wr, 305 op/s 2024-09-20T18:38:20.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 245288 9.18 MB/sec execute 551 sec latency 54.284 ms 2024-09-20T18:38:21.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 245519 9.18 MB/sec execute 552 sec latency 51.910 ms 2024-09-20T18:38:22.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:21 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:22.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:21 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:22.068 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:21 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:22.069 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:21 smithi012 ceph-mon[65348]: pgmap v39: 97 pgs: 97 active+clean; 360 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 482 op/s 2024-09-20T18:38:22.084 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:21 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:22.084 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:21 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:22.084 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:21 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:22.084 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:21 smithi174 ceph-mon[49279]: pgmap v39: 97 pgs: 97 active+clean; 360 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 482 op/s 2024-09-20T18:38:22.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 245731 9.18 MB/sec execute 553 sec latency 60.406 ms 2024-09-20T18:38:23.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:22 smithi012 ceph-mon[65348]: pgmap v40: 97 pgs: 97 active+clean; 360 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 MiB/s wr, 331 op/s 2024-09-20T18:38:23.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 245947 9.17 MB/sec execute 554 sec latency 80.784 ms 2024-09-20T18:38:23.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:22 smithi174 ceph-mon[49279]: pgmap v40: 97 pgs: 97 active+clean; 360 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 MiB/s wr, 331 op/s 2024-09-20T18:38:24.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 246122 9.16 MB/sec execute 555 sec latency 93.104 ms 2024-09-20T18:38:24.693 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:25.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:25.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 246364 9.15 MB/sec execute 556 sec latency 99.935 ms 2024-09-20T18:38:26.030 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:26.031 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:25 smithi012 ceph-mon[65348]: pgmap v41: 97 pgs: 97 active+clean; 359 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 459 op/s 2024-09-20T18:38:26.031 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:26.031 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:26.031 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:26.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 246530 9.14 MB/sec execute 557 sec latency 99.296 ms 2024-09-20T18:38:26.159 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:26.159 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:25 smithi174 ceph-mon[49279]: pgmap v41: 97 pgs: 97 active+clean; 359 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.3 MiB/s wr, 459 op/s 2024-09-20T18:38:26.159 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:26.159 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:26.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:27.048 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:26 smithi012 ceph-mon[65348]: pgmap v42: 97 pgs: 97 active+clean; 359 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.8 MiB/s wr, 350 op/s 2024-09-20T18:38:27.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 246694 9.13 MB/sec execute 558 sec latency 68.361 ms 2024-09-20T18:38:27.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:26 smithi174 ceph-mon[49279]: pgmap v42: 97 pgs: 97 active+clean; 359 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.8 MiB/s wr, 350 op/s 2024-09-20T18:38:28.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 246878 9.12 MB/sec execute 559 sec latency 73.161 ms 2024-09-20T18:38:29.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 247128 9.12 MB/sec execute 560 sec latency 88.682 ms 2024-09-20T18:38:29.368 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:29 smithi012 ceph-mon[65348]: pgmap v43: 97 pgs: 97 active+clean; 359 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.4 MiB/s wr, 317 op/s 2024-09-20T18:38:29.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.369 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:29 smithi174 ceph-mon[49279]: pgmap v43: 97 pgs: 97 active+clean; 359 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.4 MiB/s wr, 317 op/s 2024-09-20T18:38:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:29.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:30.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 247313 9.11 MB/sec execute 561 sec latency 97.840 ms 2024-09-20T18:38:30.996 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:30 smithi012 ceph-mon[65348]: pgmap v44: 97 pgs: 97 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 449 op/s 2024-09-20T18:38:31.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 247610 9.10 MB/sec execute 562 sec latency 64.496 ms 2024-09-20T18:38:31.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:30 smithi174 ceph-mon[49279]: pgmap v44: 97 pgs: 97 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 449 op/s 2024-09-20T18:38:32.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 247800 9.09 MB/sec execute 563 sec latency 65.207 ms 2024-09-20T18:38:33.065 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:32 smithi012 ceph-mon[65348]: pgmap v45: 97 pgs: 97 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.5 MiB/s wr, 272 op/s 2024-09-20T18:38:33.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 247976 9.08 MB/sec execute 564 sec latency 65.348 ms 2024-09-20T18:38:33.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:32 smithi174 ceph-mon[49279]: pgmap v45: 97 pgs: 97 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.5 MiB/s wr, 272 op/s 2024-09-20T18:38:34.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 248163 9.09 MB/sec execute 565 sec latency 104.716 ms 2024-09-20T18:38:35.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 248749 9.10 MB/sec execute 566 sec latency 113.227 ms 2024-09-20T18:38:35.853 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: pgmap v46: 97 pgs: 97 active+clean; 358 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 410 op/s 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:35.912 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi012"} : dispatch 2024-09-20T18:38:35.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi012"}]': finished 2024-09-20T18:38:35.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"} : dispatch 2024-09-20T18:38:35.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]': finished 2024-09-20T18:38:35.913 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-20T18:38:36.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 249115 9.11 MB/sec execute 567 sec latency 47.054 ms 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: pgmap v46: 97 pgs: 97 active+clean; 358 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 410 op/s 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:36.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi012"} : dispatch 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi012"}]': finished 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"} : dispatch 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi174"}]': finished 2024-09-20T18:38:36.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-20T18:38:36.921 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:36 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all crash 2024-09-20T18:38:36.921 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:36 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T18:38:36.921 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:36 smithi012 ceph-mon[65348]: Upgrade: osd.0 is safe to restart 2024-09-20T18:38:36.921 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:36 smithi012 ceph-mon[65348]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:37.016 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:38:37.016 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (3m) 8s ago 18m 15.9M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:38:37.016 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (23s) 8s ago 18m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:38:37.016 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (16s) 12s ago 17m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (3m) 8s ago 17m 44.3M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (12m) 8s ago 12m 40.8M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (11m) 12s ago 11m 15.9M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (2m) 8s ago 19m 512M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (113s) 12s ago 17m 454M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (96s) 8s ago 19m 42.9M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (71s) 12s ago 17m 36.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (11m) 8s ago 11m 288M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (11m) 12s ago 11m 83.9M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (3m) 8s ago 17m 8552k - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:38:37.017 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (2m) 12s ago 17m 11.9M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (16m) 8s ago 16m 555M 1937M 17.2.0 e1d6a67b021e 9a337fa113c8 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (15m) 8s ago 15m 449M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (15m) 8s ago 15m 389M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (15m) 8s ago 15m 470M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (14m) 12s ago 14m 463M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (14m) 12s ago 14m 578M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (13m) 12s ago 13m 509M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (13m) 12s ago 13m 765M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:38:37.018 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (119s) 8s ago 17m 48.6M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:38:37.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 249424 9.11 MB/sec execute 568 sec latency 81.724 ms 2024-09-20T18:38:37.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:36 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all crash 2024-09-20T18:38:37.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:36 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-20T18:38:37.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:36 smithi174 ceph-mon[49279]: Upgrade: osd.0 is safe to restart 2024-09-20T18:38:37.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:36 smithi174 ceph-mon[49279]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:37.702 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:38:37.703 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:38:37.704 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:38:37.704 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-20T18:38:37.704 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 4 2024-09-20T18:38:37.704 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:38:37.704 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:38:37.953 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:37 smithi012 ceph-mon[65348]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:37.953 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:37 smithi012 ceph-mon[65348]: pgmap v47: 97 pgs: 97 active+clean; 360 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.1 MiB/s wr, 296 op/s 2024-09-20T18:38:37.954 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:37 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/3228630015' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:38.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 249646 9.10 MB/sec execute 569 sec latency 64.195 ms 2024-09-20T18:38:38.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:37 smithi174 ceph-mon[49279]: from='client.44129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:38.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:37 smithi174 ceph-mon[49279]: pgmap v47: 97 pgs: 97 active+clean; 360 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.1 MiB/s wr, 296 op/s 2024-09-20T18:38:38.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:37 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/3228630015' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:38:38.280 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "6/23 daemons upgraded", 2024-09-20T18:38:38.281 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:38:38.282 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:38:38.282 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:38:38.964 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:38 smithi012 ceph-mon[65348]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:38.964 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:38.964 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T18:38:38.965 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:38.969 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:38:39.148 INFO:teuthology.orchestra.run.smithi012.stdout: 5 249901 9.09 MB/sec execute 570 sec latency 97.229 ms 2024-09-20T18:38:39.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:38 smithi174 ceph-mon[49279]: from='client.44133 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:39.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:39.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-20T18:38:39.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:38:39.985 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:39 smithi012 ceph-mon[65348]: Upgrade: Updating osd.0 2024-09-20T18:38:39.986 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:39 smithi012 ceph-mon[65348]: Deploying daemon osd.0 on smithi012 2024-09-20T18:38:39.986 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:39 smithi012 ceph-mon[65348]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:39.986 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:39 smithi012 ceph-mon[65348]: pgmap v48: 97 pgs: 97 active+clean; 360 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.8 MiB/s wr, 284 op/s 2024-09-20T18:38:39.986 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:39 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/1201524692' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:38:39.986 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:39 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:40.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 250124 9.08 MB/sec execute 571 sec latency 54.985 ms 2024-09-20T18:38:40.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:39 smithi174 ceph-mon[49279]: Upgrade: Updating osd.0 2024-09-20T18:38:40.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:39 smithi174 ceph-mon[49279]: Deploying daemon osd.0 on smithi012 2024-09-20T18:38:40.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:39 smithi174 ceph-mon[49279]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:38:40.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:39 smithi174 ceph-mon[49279]: pgmap v48: 97 pgs: 97 active+clean; 360 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.1 KiB/s rd, 2.8 MiB/s wr, 284 op/s 2024-09-20T18:38:40.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:39 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/1201524692' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:38:40.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:39 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:40.993 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:40 smithi012 ceph-mon[65348]: pgmap v49: 97 pgs: 97 active+clean; 361 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-20T18:38:41.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 250323 9.08 MB/sec execute 572 sec latency 69.067 ms 2024-09-20T18:38:41.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:40 smithi174 ceph-mon[49279]: pgmap v49: 97 pgs: 97 active+clean; 361 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-20T18:38:42.000 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:41 smithi012 ceph-mon[65348]: mgrmap e43: smithi012.xkxxrd(active, since 92s), standbys: smithi174.nxkptu 2024-09-20T18:38:42.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 250530 9.07 MB/sec execute 573 sec latency 156.769 ms 2024-09-20T18:38:42.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:41 smithi174 ceph-mon[49279]: mgrmap e43: smithi012.xkxxrd(active, since 92s), standbys: smithi174.nxkptu 2024-09-20T18:38:43.028 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:42 smithi012 ceph-mon[65348]: pgmap v50: 97 pgs: 97 active+clean; 361 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 341 op/s 2024-09-20T18:38:43.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 250819 9.07 MB/sec execute 574 sec latency 82.138 ms 2024-09-20T18:38:43.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:42 smithi174 ceph-mon[49279]: pgmap v50: 97 pgs: 97 active+clean; 361 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 341 op/s 2024-09-20T18:38:43.327 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:43 smithi012 systemd[1]: Stopping Ceph osd.0 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:38:43.837 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:43 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[34367]: 2024-09-20T18:38:43.587+0000 7fdf7c81b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:38:43.837 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:43 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[34367]: 2024-09-20T18:38:43.587+0000 7fdf7c81b700 -1 osd.0 73 *** Got signal Terminated *** 2024-09-20T18:38:43.837 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:43 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[34367]: 2024-09-20T18:38:43.587+0000 7fdf7c81b700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:38:44.089 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:43 smithi012 ceph-mon[65348]: osd.0 marked itself down and dead 2024-09-20T18:38:44.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 251087 9.06 MB/sec execute 575 sec latency 174.182 ms 2024-09-20T18:38:44.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:43 smithi174 ceph-mon[49279]: osd.0 marked itself down and dead 2024-09-20T18:38:44.880 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:44 smithi012 podman[69628]: 2024-09-20 18:38:44.593172459 +0000 UTC m=+1.147663291 container died 9a337fa113c8dd0f6d8ebf9aae982a00031b005eaad5c0a8c277483e6e40504d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, release=754, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , distribution-scope=public, io.openshift.tags=base centos centos-stream) 2024-09-20T18:38:45.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 251128 9.04 MB/sec execute 576 sec latency 1037.188 ms 2024-09-20T18:38:45.152 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:44 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:38:45.152 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:44 smithi012 ceph-mon[65348]: osdmap e74: 8 total, 7 up, 8 in 2024-09-20T18:38:45.153 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:44 smithi012 ceph-mon[65348]: pgmap v52: 97 pgs: 22 peering, 4 stale+active+clean, 71 active+clean; 359 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.5 MiB/s wr, 352 op/s 2024-09-20T18:38:45.153 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:44 smithi012 podman[69628]: 2024-09-20 18:38:44.981066452 +0000 UTC m=+1.535557290 container cleanup 9a337fa113c8dd0f6d8ebf9aae982a00031b005eaad5c0a8c277483e6e40504d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., version=8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, release=754, vcs-type=git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_CLEAN=True) 2024-09-20T18:38:45.153 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:44 smithi012 bash[69628]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0 2024-09-20T18:38:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:44 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:38:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:44 smithi174 ceph-mon[49279]: osdmap e74: 8 total, 7 up, 8 in 2024-09-20T18:38:45.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:44 smithi174 ceph-mon[49279]: pgmap v52: 97 pgs: 22 peering, 4 stale+active+clean, 71 active+clean; 359 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 3.5 MiB/s wr, 352 op/s 2024-09-20T18:38:45.655 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:45 smithi012 podman[69644]: 2024-09-20 18:38:45.517856834 +0000 UTC m=+0.924106386 container remove 9a337fa113c8dd0f6d8ebf9aae982a00031b005eaad5c0a8c277483e6e40504d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, distribution-scope=public, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64) 2024-09-20T18:38:46.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 251354 9.04 MB/sec execute 577 sec latency 92.002 ms 2024-09-20T18:38:46.176 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:45 smithi012 ceph-mon[65348]: Health check failed: Reduced data availability: 2 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-20T18:38:46.176 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:45 smithi012 ceph-mon[65348]: osdmap e75: 8 total, 7 up, 8 in 2024-09-20T18:38:46.176 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:46 smithi012 podman[69723]: 2024-09-20 18:38:46.019899761 +0000 UTC m=+0.023616427 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:38:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:45 smithi174 ceph-mon[49279]: Health check failed: Reduced data availability: 2 pgs inactive, 8 pgs peering (PG_AVAILABILITY) 2024-09-20T18:38:46.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:45 smithi174 ceph-mon[49279]: osdmap e75: 8 total, 7 up, 8 in 2024-09-20T18:38:46.464 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:46 smithi012 podman[69723]: 2024-09-20 18:38:46.23389221 +0000 UTC m=+0.237608874 container create d135067079a3f0a10e47b5a5a99953bacc96877f92138bf8003e9c3d6eef68b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-20T18:38:46.725 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:46 smithi012 podman[69723]: 2024-09-20 18:38:46.660143257 +0000 UTC m=+0.663859915 container init d135067079a3f0a10e47b5a5a99953bacc96877f92138bf8003e9c3d6eef68b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-20T18:38:46.725 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:46 smithi012 podman[69723]: 2024-09-20 18:38:46.666384908 +0000 UTC m=+0.670101574 container start d135067079a3f0a10e47b5a5a99953bacc96877f92138bf8003e9c3d6eef68b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-20T18:38:47.007 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:46 smithi012 ceph-mon[65348]: pgmap v54: 97 pgs: 36 peering, 61 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.4 MiB/s wr, 479 op/s 2024-09-20T18:38:47.007 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:46 smithi012 podman[69723]: 2024-09-20 18:38:46.848697143 +0000 UTC m=+0.852413819 container attach d135067079a3f0a10e47b5a5a99953bacc96877f92138bf8003e9c3d6eef68b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, org.label-schema.build-date=20240731) 2024-09-20T18:38:47.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 251579 9.04 MB/sec execute 578 sec latency 64.437 ms 2024-09-20T18:38:47.263 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:47 smithi012 podman[69723]: 2024-09-20 18:38:47.004536408 +0000 UTC m=+1.008253074 container died d135067079a3f0a10e47b5a5a99953bacc96877f92138bf8003e9c3d6eef68b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:38:47.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:46 smithi174 ceph-mon[49279]: pgmap v54: 97 pgs: 36 peering, 61 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.4 MiB/s wr, 479 op/s 2024-09-20T18:38:48.036 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:47 smithi012 podman[69746]: 2024-09-20 18:38:47.808256423 +0000 UTC m=+0.795984844 container remove d135067079a3f0a10e47b5a5a99953bacc96877f92138bf8003e9c3d6eef68b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-20T18:38:48.036 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:47 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-20T18:38:48.036 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:47 smithi012 systemd[1]: Stopped Ceph osd.0 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:38:48.036 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:47 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.0.service: Consumed 5min 31.745s CPU time. 2024-09-20T18:38:48.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 251803 9.04 MB/sec execute 579 sec latency 59.795 ms 2024-09-20T18:38:48.286 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 systemd[1]: Starting Ceph osd.0 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:38:48.547 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 podman[69843]: 2024-09-20 18:38:48.323644854 +0000 UTC m=+0.028072627 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:38:48.548 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 podman[69843]: 2024-09-20 18:38:48.455874203 +0000 UTC m=+0.160301973 container create 87ee907372a0e387d5995dcfc5f026f9e7e569969fbb91355c33961faae7f706 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:38:48.813 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 podman[69843]: 2024-09-20 18:38:48.750990209 +0000 UTC m=+0.455417988 container init 87ee907372a0e387d5995dcfc5f026f9e7e569969fbb91355c33961faae7f706 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-20T18:38:48.813 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 podman[69843]: 2024-09-20 18:38:48.760415183 +0000 UTC m=+0.464842961 container start 87ee907372a0e387d5995dcfc5f026f9e7e569969fbb91355c33961faae7f706 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T18:38:49.064 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:48 smithi012 ceph-mon[65348]: pgmap v55: 97 pgs: 36 peering, 61 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.5 MiB/s wr, 196 op/s 2024-09-20T18:38:49.064 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 podman[69843]: 2024-09-20 18:38:48.894177366 +0000 UTC m=+0.598605146 container attach 87ee907372a0e387d5995dcfc5f026f9e7e569969fbb91355c33961faae7f706 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-20T18:38:49.065 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:49.065 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 bash[69843]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:49.065 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:49.065 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:48 smithi012 bash[69843]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:49.149 INFO:teuthology.orchestra.run.smithi012.stdout: 5 252148 9.04 MB/sec execute 580 sec latency 52.161 ms 2024-09-20T18:38:49.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:48 smithi174 ceph-mon[49279]: pgmap v55: 97 pgs: 36 peering, 61 active+clean; 355 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.5 MiB/s wr, 196 op/s 2024-09-20T18:38:50.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 252468 9.04 MB/sec execute 581 sec latency 51.771 ms 2024-09-20T18:38:51.118 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:50 smithi012 ceph-mon[65348]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 362 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 428 op/s; 215/1653 objects degraded (13.007%) 2024-09-20T18:38:51.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 252767 9.04 MB/sec execute 582 sec latency 61.355 ms 2024-09-20T18:38:51.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:50 smithi174 ceph-mon[49279]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 362 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 428 op/s; 215/1653 objects degraded (13.007%) 2024-09-20T18:38:52.128 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:51 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 215/1653 objects degraded (13.007%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:38:52.129 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:51 smithi012 ceph-mon[65348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 16 pgs peering) 2024-09-20T18:38:52.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 253195 9.05 MB/sec execute 583 sec latency 33.990 ms 2024-09-20T18:38:52.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:51 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 215/1653 objects degraded (13.007%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:38:52.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:51 smithi174 ceph-mon[49279]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 16 pgs peering) 2024-09-20T18:38:53.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 253634 9.04 MB/sec execute 584 sec latency 41.466 ms 2024-09-20T18:38:53.645 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:38:53.645 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:53.646 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 bash[69843]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:38:53.646 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 bash[69843]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:53.907 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:53.907 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 bash[69843]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:38:53.908 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T18:38:53.908 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 bash[69843]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T18:38:53.908 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-20T18:38:53.908 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:53 smithi012 bash[69843]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-09-20T18:38:54.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 254179 9.06 MB/sec execute 585 sec latency 34.478 ms 2024-09-20T18:38:54.158 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:53 smithi012 ceph-mon[65348]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 362 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 382 op/s; 215/1653 objects degraded (13.007%) 2024-09-20T18:38:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:53 smithi174 ceph-mon[49279]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 362 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.8 MiB/s wr, 382 op/s; 215/1653 objects degraded (13.007%) 2024-09-20T18:38:54.409 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 bash[69843]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 bash[69843]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 bash[69843]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 bash[69843]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate[69857]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 bash[69843]: --> ceph-volume lvm activate successful for osd ID: 0 2024-09-20T18:38:54.410 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:54 smithi012 podman[70090]: 2024-09-20 18:38:54.287691985 +0000 UTC m=+0.026144486 container died 87ee907372a0e387d5995dcfc5f026f9e7e569969fbb91355c33961faae7f706 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True) 2024-09-20T18:38:55.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 254872 9.06 MB/sec execute 586 sec latency 37.002 ms 2024-09-20T18:38:55.428 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:55 smithi012 podman[70090]: 2024-09-20 18:38:55.215839367 +0000 UTC m=+0.954291860 container remove 87ee907372a0e387d5995dcfc5f026f9e7e569969fbb91355c33961faae7f706 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-activate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-20T18:38:55.700 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:55 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:55.700 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:55 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:55.701 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:55 smithi012 podman[70147]: 2024-09-20 18:38:55.399875412 +0000 UTC m=+0.027269002 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:38:55.701 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:55 smithi012 podman[70147]: 2024-09-20 18:38:55.613915936 +0000 UTC m=+0.241309523 container create 280ecc2705828fb8abc8af338c0eff056dbdeceb6862d6b9cbb99f39c3f8d8ad (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-20T18:38:56.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:55 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:56.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:55 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:38:56.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 255234 9.07 MB/sec execute 587 sec latency 80.771 ms 2024-09-20T18:38:56.213 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:55 smithi012 podman[70147]: 2024-09-20 18:38:55.964195019 +0000 UTC m=+0.591588597 container init 280ecc2705828fb8abc8af338c0eff056dbdeceb6862d6b9cbb99f39c3f8d8ad (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-20T18:38:56.213 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:55 smithi012 podman[70147]: 2024-09-20 18:38:55.970044383 +0000 UTC m=+0.597437960 container start 280ecc2705828fb8abc8af338c0eff056dbdeceb6862d6b9cbb99f39c3f8d8ad (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-20T18:38:56.213 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:55 smithi012 ceph-osd[70165]: -- 172.21.15.12:0/1093140674 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5630c4f6a1e0 con 0x5630c4105c00 2024-09-20T18:38:56.214 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:56 smithi012 bash[70147]: 280ecc2705828fb8abc8af338c0eff056dbdeceb6862d6b9cbb99f39c3f8d8ad 2024-09-20T18:38:56.214 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:56 smithi012 systemd[1]: Started Ceph osd.0 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:38:56.717 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:56 smithi012 ceph-mon[65348]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 364 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 529 op/s; 223/1719 objects degraded (12.973%) 2024-09-20T18:38:56.717 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:56 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 223/1719 objects degraded (12.973%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:38:56.717 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:56.717 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:56.717 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:57.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:56 smithi174 ceph-mon[49279]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 364 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 529 op/s; 223/1719 objects degraded (12.973%) 2024-09-20T18:38:57.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:56 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 223/1719 objects degraded (12.973%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:38:57.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:57.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:38:57.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:38:57.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 255645 9.07 MB/sec execute 588 sec latency 40.548 ms 2024-09-20T18:38:58.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 255974 9.08 MB/sec execute 589 sec latency 48.644 ms 2024-09-20T18:38:58.232 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:38:58 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[70161]: 2024-09-20T18:38:58.122+0000 7f6d18c26640 -1 Falling back to public interface 2024-09-20T18:38:58.744 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:38:58 smithi012 ceph-mon[65348]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 363 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 440 op/s; 219/1689 objects degraded (12.966%) 2024-09-20T18:38:59.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:38:58 smithi174 ceph-mon[49279]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 363 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.5 MiB/s wr, 440 op/s; 219/1689 objects degraded (12.966%) 2024-09-20T18:38:59.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 256296 9.08 MB/sec execute 590 sec latency 62.560 ms 2024-09-20T18:39:00.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 256430 9.07 MB/sec execute 591 sec latency 56.464 ms 2024-09-20T18:39:00.812 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:00 smithi012 ceph-mon[65348]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 363 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 219/1689 objects degraded (12.966%) 2024-09-20T18:39:01.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:00 smithi174 ceph-mon[49279]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 363 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.5 MiB/s wr, 438 op/s; 219/1689 objects degraded (12.966%) 2024-09-20T18:39:01.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 256638 9.07 MB/sec execute 592 sec latency 59.170 ms 2024-09-20T18:39:01.901 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:01 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 219/1689 objects degraded (12.966%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:02.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:01 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 219/1689 objects degraded (12.966%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:02.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 256835 9.06 MB/sec execute 593 sec latency 114.130 ms 2024-09-20T18:39:02.669 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:02 smithi012 ceph-mon[65348]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 360 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 653 op/s; 209/1623 objects degraded (12.877%) 2024-09-20T18:39:03.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:02 smithi174 ceph-mon[49279]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 360 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 8.6 KiB/s rd, 7.0 MiB/s wr, 653 op/s; 209/1623 objects degraded (12.877%) 2024-09-20T18:39:03.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 257067 9.05 MB/sec execute 594 sec latency 59.587 ms 2024-09-20T18:39:04.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 257289 9.04 MB/sec execute 595 sec latency 58.393 ms 2024-09-20T18:39:04.779 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:04 smithi012 ceph-mon[65348]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 360 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 498 op/s; 209/1623 objects degraded (12.877%) 2024-09-20T18:39:04.779 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:04 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:04.779 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:04 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:05.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:04 smithi174 ceph-mon[49279]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 360 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 498 op/s; 209/1623 objects degraded (12.877%) 2024-09-20T18:39:05.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:04 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:05.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:04 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:05.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 257586 9.04 MB/sec execute 596 sec latency 50.531 ms 2024-09-20T18:39:06.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 257979 9.04 MB/sec execute 597 sec latency 56.171 ms 2024-09-20T18:39:06.294 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:06 smithi012 ceph-mon[65348]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 358 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 632 op/s; 193/1557 objects degraded (12.396%) 2024-09-20T18:39:06.294 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:06 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:06.294 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:06 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:06.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:06 smithi174 ceph-mon[49279]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 358 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 632 op/s; 193/1557 objects degraded (12.396%) 2024-09-20T18:39:06.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:06 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:06.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:06 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:07.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 258387 9.04 MB/sec execute 598 sec latency 53.995 ms 2024-09-20T18:39:07.311 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:07 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 193/1557 objects degraded (12.396%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:07.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:07 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 193/1557 objects degraded (12.396%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:08.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 258662 9.04 MB/sec execute 599 sec latency 42.179 ms 2024-09-20T18:39:08.383 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:08 smithi012 ceph-mon[65348]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 401 op/s; 186/1512 objects degraded (12.302%) 2024-09-20T18:39:08.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:08 smithi174 ceph-mon[49279]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 401 op/s; 186/1512 objects degraded (12.302%) 2024-09-20T18:39:08.637 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:39:08 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[70161]: 2024-09-20T18:39:08.407+0000 7f6d18c26640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-20T18:39:09.150 INFO:teuthology.orchestra.run.smithi012.stdout: 5 cleanup 600 sec 2024-09-20T18:39:09.322 INFO:teuthology.orchestra.run.smithi012.stdout:[260985] unlink './clients/client4/~dmtmp/PARADOX/.nfs00000100000002e800000006' failed - Device or resource busy 2024-09-20T18:39:09.323 INFO:teuthology.orchestra.run.smithi012.stdout:[260985] unlink './clients/client4/~dmtmp/PARADOX/.nfs00000100000002c600000004' failed - Device or resource busy 2024-09-20T18:39:09.324 INFO:teuthology.orchestra.run.smithi012.stdout:[260985] unlink './clients/client4/~dmtmp/PARADOX/.nfs00000100000002f200000009' failed - Device or resource busy 2024-09-20T18:39:09.324 INFO:teuthology.orchestra.run.smithi012.stdout:[260985] unlink './clients/client4/~dmtmp/PARADOX/.nfs00000100000002d20000000d' failed - Device or resource busy 2024-09-20T18:39:09.326 INFO:teuthology.orchestra.run.smithi012.stdout:[260985] unlink './clients/client4/~dmtmp/PARADOX/.nfs00000100000002c10000000c' failed - Device or resource busy 2024-09-20T18:39:09.327 INFO:teuthology.orchestra.run.smithi012.stdout:[260985] unlink './clients/client4/~dmtmp/PARADOX/.nfs00000100000002cc00000005' failed - Device or resource busy 2024-09-20T18:39:09.341 INFO:teuthology.orchestra.run.smithi012.stdout:[260985] unlink './clients/client4/~dmtmp/PARADOX/.nfs00000100000002dd00000007' failed - Device or resource busy 2024-09-20T18:39:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: 0 cleanup 600 sec 2024-09-20T18:39:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:39:09.521 INFO:teuthology.orchestra.run.smithi012.stdout: Operation Count AvgLat MaxLat 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: ---------------------------------------- 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: NTCreateX 174110 4.978 211.624 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Close 127980 0.144 25.028 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Rename 7375 9.894 123.236 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Unlink 35079 7.517 146.508 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Qpathinfo 157986 1.269 121.238 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Qfileinfo 27570 0.007 1.833 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Qfsinfo 28870 1.566 110.732 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Sfileinfo 14223 13.962 135.125 2024-09-20T18:39:09.522 INFO:teuthology.orchestra.run.smithi012.stdout: Find 60986 1.033 28.656 2024-09-20T18:39:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: WriteX 86066 14.691 1037.180 2024-09-20T18:39:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: ReadX 272938 0.010 3.879 2024-09-20T18:39:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: LockX 566 0.394 6.909 2024-09-20T18:39:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: UnlockX 566 0.343 1.928 2024-09-20T18:39:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: Flush 12199 0.003 0.042 2024-09-20T18:39:09.523 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:39:09.523 INFO:teuthology.orchestra.run.smithi012.stdout:Throughput 9.03877 MB/sec 5 clients 5 procs max_latency=1037.188 ms 2024-09-20T18:39:09.524 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-20T18:39:09.566 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-20T18:39:09.629 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:39:09.716 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-20T18:39:09.783 INFO:teuthology.orchestra.run.smithi012.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-20T18:39:09.783 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:39:09.784 INFO:teuthology.orchestra.run.smithi012.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-20T18:39:09.784 INFO:teuthology.orchestra.run.smithi012.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-20T18:39:09.884 INFO:teuthology.orchestra.run.smithi012.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-20T18:39:09.885 INFO:teuthology.orchestra.run.smithi012.stdout:releasing clients 2024-09-20T18:39:10.169 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:39:10 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[70161]: 2024-09-20T18:39:10.055+0000 7f6d18c26640 -1 osd.0 73 log_to_monitors true 2024-09-20T18:39:10.430 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:39:10 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[70161]: 2024-09-20T18:39:10.252+0000 7f6d109d0640 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:39:10.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:10 smithi012 ceph-mon[65348]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 364 op/s; 186/1512 objects degraded (12.302%) 2024-09-20T18:39:10.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:10.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:10 smithi012 ceph-mon[65348]: from='client.44149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:10.431 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:10 smithi012 ceph-mon[65348]: from='osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-20T18:39:10.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:10 smithi174 ceph-mon[49279]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.8 MiB/s wr, 364 op/s; 186/1512 objects degraded (12.302%) 2024-09-20T18:39:10.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:10.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:10 smithi174 ceph-mon[49279]: from='client.44149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:10.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:10 smithi174 ceph-mon[49279]: from='osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-20T18:39:10.628 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:39:10.628 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (4m) 6s ago 18m 15.9M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (56s) 6s ago 18m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (50s) 45s ago 18m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (4m) 6s ago 18m 44.0M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (12m) 6s ago 12m 41.6M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (12m) 45s ago 12m 15.9M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (3m) 6s ago 19m 515M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (2m) 45s ago 17m 454M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (2m) 6s ago 19m 48.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (104s) 45s ago 17m 36.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (12m) 6s ago 12m 289M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (12m) 45s ago 12m 83.9M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (3m) 6s ago 18m 9.94M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:39:10.629 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (3m) 45s ago 17m 11.9M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (15s) 6s ago 17m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (16m) 6s ago 16m 468M 1937M 17.2.0 e1d6a67b021e 8d6c45aab0b4 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (16m) 6s ago 16m 419M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (15m) 6s ago 15m 510M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (15m) 45s ago 15m 463M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (14m) 45s ago 14m 578M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (14m) 45s ago 14m 509M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (14m) 45s ago 14m 765M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:39:10.630 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (2m) 6s ago 17m 48.7M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:39:11.297 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 5 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:39:11.298 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:39:11.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:11 smithi012 ceph-mon[65348]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:11.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:11 smithi012 ceph-mon[65348]: from='osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T18:39:11.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:11 smithi012 ceph-mon[65348]: osdmap e76: 8 total, 7 up, 8 in 2024-09-20T18:39:11.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:11 smithi012 ceph-mon[65348]: from='osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:39:11.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:11 smithi012 ceph-mon[65348]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:11.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:11 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 186/1512 objects degraded (12.302%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:11.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:11 smithi174 ceph-mon[49279]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:11.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:11 smithi174 ceph-mon[49279]: from='osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-20T18:39:11.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:11 smithi174 ceph-mon[49279]: osdmap e76: 8 total, 7 up, 8 in 2024-09-20T18:39:11.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:11 smithi174 ceph-mon[49279]: from='osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:39:11.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:11 smithi174 ceph-mon[49279]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:11.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:11 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 186/1512 objects degraded (12.302%), 29 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:11.696 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:39:11.696 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "7/23 daemons upgraded", 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:39:11.697 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:39:11.885 INFO:teuthology.orchestra.run.smithi012.stdout: 5 283 10.66 MB/sec execute 1 sec latency 676.724 ms 2024-09-20T18:39:12.139 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN Degraded data redundancy: 198/1548 objects degraded (12.791%), 29 pgs degraded 2024-09-20T18:39:12.139 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 198/1548 objects degraded (12.791%), 29 pgs degraded 2024-09-20T18:39:12.139 INFO:teuthology.orchestra.run.smithi012.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-20T18:39:12.139 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-20T18:39:12.139 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.5 is active+undersized+degraded, acting [2,4] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-20T18:39:12.140 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-20T18:39:12.141 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.5 MiB/s wr, 388 op/s; 198/1548 objects degraded (12.791%) 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395] boot 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2257244887' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:39:12.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/3684811842' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:39:12.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:12 smithi012 ceph-mon[65348]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.5 MiB/s wr, 388 op/s; 198/1548 objects degraded (12.791%) 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: osd.0 [v2:172.21.15.12:6802/1703498395,v1:172.21.15.12:6803/1703498395] boot 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: osdmap e77: 8 total, 8 up, 8 in 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2257244887' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:12.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='client.34220 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/3684811842' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:39:12.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:12 smithi174 ceph-mon[49279]: osdmap e78: 8 total, 8 up, 8 in 2024-09-20T18:39:12.885 INFO:teuthology.orchestra.run.smithi012.stdout: 5 283 5.33 MB/sec execute 2 sec latency 1676.773 ms 2024-09-20T18:39:13.885 INFO:teuthology.orchestra.run.smithi012.stdout: 5 352 6.94 MB/sec execute 3 sec latency 2153.578 ms 2024-09-20T18:39:14.376 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:14 smithi012 ceph-mon[65348]: pgmap v70: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.3 MiB/s wr, 348 op/s; 198/1548 objects degraded (12.791%) 2024-09-20T18:39:14.376 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:14 smithi012 ceph-mon[65348]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T18:39:14.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:14 smithi174 ceph-mon[49279]: pgmap v70: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 356 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.3 MiB/s wr, 348 op/s; 198/1548 objects degraded (12.791%) 2024-09-20T18:39:14.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:14 smithi174 ceph-mon[49279]: osdmap e79: 8 total, 8 up, 8 in 2024-09-20T18:39:14.885 INFO:teuthology.orchestra.run.smithi012.stdout: 5 487 10.66 MB/sec execute 4 sec latency 30.442 ms 2024-09-20T18:39:15.885 INFO:teuthology.orchestra.run.smithi012.stdout: 5 cleanup 5 sec 2024-09-20T18:39:16.115 INFO:teuthology.orchestra.run.smithi012.stdout: 0 cleanup 5 sec 2024-09-20T18:39:16.115 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:39:16.115 INFO:teuthology.orchestra.run.smithi012.stdout: Operation Count AvgLat MaxLat 2024-09-20T18:39:16.115 INFO:teuthology.orchestra.run.smithi012.stdout: ---------------------------------------- 2024-09-20T18:39:16.115 INFO:teuthology.orchestra.run.smithi012.stdout: NTCreateX 291 15.929 59.857 2024-09-20T18:39:16.115 INFO:teuthology.orchestra.run.smithi012.stdout: Close 284 42.989 2135.916 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Rename 4 30.621 34.762 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Unlink 9 11.431 27.907 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Qpathinfo 36 0.789 5.634 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Qfileinfo 268 26.583 2153.570 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Qfsinfo 42 0.603 1.584 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Sfileinfo 18 18.137 38.031 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Find 38 0.971 1.804 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: WriteX 1271 0.030 0.071 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: ReadX 135 0.015 0.289 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: Flush 29 16.065 27.114 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:39:16.116 INFO:teuthology.orchestra.run.smithi012.stdout:Throughput 10.6636 MB/sec 5 clients 5 procs max_latency=2153.578 ms 2024-09-20T18:39:16.117 DEBUG:teuthology.parallel:result is None 2024-09-20T18:39:16.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:16 smithi012 ceph-mon[65348]: pgmap v72: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized, 15 active+undersized+degraded, 70 active+clean; 339 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 10 MiB/s wr, 123 op/s; 267/1335 objects degraded (20.000%); 0 B/s, 0 objects/s recovering 2024-09-20T18:39:16.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:16 smithi012 ceph-mon[65348]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T18:39:16.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:16 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 267/1335 objects degraded (20.000%), 21 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:16.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:16 smithi012 ceph-mon[65348]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T18:39:16.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:16 smithi174 ceph-mon[49279]: pgmap v72: 97 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized, 15 active+undersized+degraded, 70 active+clean; 339 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 8.9 KiB/s rd, 10 MiB/s wr, 123 op/s; 267/1335 objects degraded (20.000%); 0 B/s, 0 objects/s recovering 2024-09-20T18:39:16.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:16 smithi174 ceph-mon[49279]: osdmap e80: 8 total, 8 up, 8 in 2024-09-20T18:39:16.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:16 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 267/1335 objects degraded (20.000%), 21 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:16.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:16 smithi174 ceph-mon[49279]: osdmap e81: 8 total, 8 up, 8 in 2024-09-20T18:39:18.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:18 smithi012 ceph-mon[65348]: pgmap v75: 97 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 329 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 9.4 KiB/s rd, 18 MiB/s wr, 381 op/s; 350/1209 objects degraded (28.950%); 19 B/s, 8 objects/s recovering 2024-09-20T18:39:18.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:18 smithi012 ceph-mon[65348]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T18:39:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:18 smithi174 ceph-mon[49279]: pgmap v75: 97 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 329 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 9.4 KiB/s rd, 18 MiB/s wr, 381 op/s; 350/1209 objects degraded (28.950%); 19 B/s, 8 objects/s recovering 2024-09-20T18:39:18.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:18 smithi174 ceph-mon[49279]: osdmap e82: 8 total, 8 up, 8 in 2024-09-20T18:39:19.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:19 smithi174 ceph-mon[49279]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T18:39:19.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:19 smithi174 ceph-mon[49279]: osdmap e84: 8 total, 8 up, 8 in 2024-09-20T18:39:19.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:19 smithi012 ceph-mon[65348]: osdmap e83: 8 total, 8 up, 8 in 2024-09-20T18:39:19.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:19 smithi012 ceph-mon[65348]: osdmap e84: 8 total, 8 up, 8 in 2024-09-20T18:39:20.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:20 smithi174 ceph-mon[49279]: pgmap v78: 97 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 329 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 511 B/s rd, 9.8 MiB/s wr, 303 op/s; 350/1209 objects degraded (28.950%); 22 B/s, 9 objects/s recovering 2024-09-20T18:39:20.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:20 smithi012 ceph-mon[65348]: pgmap v78: 97 pgs: 14 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 329 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 511 B/s rd, 9.8 MiB/s wr, 303 op/s; 350/1209 objects degraded (28.950%); 22 B/s, 9 objects/s recovering 2024-09-20T18:39:21.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:21 smithi174 ceph-mon[49279]: osdmap e85: 8 total, 8 up, 8 in 2024-09-20T18:39:21.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:21 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 350/1209 objects degraded (28.950%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:21.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:21 smithi012 ceph-mon[65348]: osdmap e85: 8 total, 8 up, 8 in 2024-09-20T18:39:21.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:21 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 350/1209 objects degraded (28.950%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:22.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:22 smithi174 ceph-mon[49279]: pgmap v81: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 255 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 190 op/s; 220/462 objects degraded (47.619%); 2.7 MiB/s, 32 objects/s recovering 2024-09-20T18:39:22.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:22 smithi174 ceph-mon[49279]: osdmap e86: 8 total, 8 up, 8 in 2024-09-20T18:39:22.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:22 smithi012 ceph-mon[65348]: pgmap v81: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 255 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 190 op/s; 220/462 objects degraded (47.619%); 2.7 MiB/s, 32 objects/s recovering 2024-09-20T18:39:22.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:22 smithi012 ceph-mon[65348]: osdmap e86: 8 total, 8 up, 8 in 2024-09-20T18:39:23.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:23 smithi174 ceph-mon[49279]: osdmap e87: 8 total, 8 up, 8 in 2024-09-20T18:39:23.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:23 smithi012 ceph-mon[65348]: osdmap e87: 8 total, 8 up, 8 in 2024-09-20T18:39:24.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:24 smithi012 ceph-mon[65348]: pgmap v84: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 255 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 190 op/s; 220/462 objects degraded (47.619%); 2.7 MiB/s, 32 objects/s recovering 2024-09-20T18:39:24.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:24 smithi012 ceph-mon[65348]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T18:39:24.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:24 smithi174 ceph-mon[49279]: pgmap v84: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 255 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.4 MiB/s wr, 190 op/s; 220/462 objects degraded (47.619%); 2.7 MiB/s, 32 objects/s recovering 2024-09-20T18:39:24.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:24 smithi174 ceph-mon[49279]: osdmap e88: 8 total, 8 up, 8 in 2024-09-20T18:39:25.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:25 smithi174 ceph-mon[49279]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T18:39:25.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:25.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:25.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:25 smithi012 ceph-mon[65348]: osdmap e89: 8 total, 8 up, 8 in 2024-09-20T18:39:25.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:25.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:26.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:26 smithi174 ceph-mon[49279]: pgmap v87: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 250 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 511 B/s rd, 767 B/s wr, 1 op/s; 220/456 objects degraded (48.246%) 2024-09-20T18:39:26.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:26 smithi174 ceph-mon[49279]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T18:39:26.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:26 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 220/456 objects degraded (48.246%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:26.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:26 smithi012 ceph-mon[65348]: pgmap v87: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 250 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 511 B/s rd, 767 B/s wr, 1 op/s; 220/456 objects degraded (48.246%) 2024-09-20T18:39:26.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:26 smithi012 ceph-mon[65348]: osdmap e90: 8 total, 8 up, 8 in 2024-09-20T18:39:26.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:26 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 220/456 objects degraded (48.246%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:27.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:27 smithi012 ceph-mon[65348]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T18:39:27.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:27 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T18:39:27.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:27 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T18:39:27.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:27 smithi012 ceph-mon[65348]: Upgrade: osd.1 is safe to restart 2024-09-20T18:39:27.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:27 smithi174 ceph-mon[49279]: osdmap e91: 8 total, 8 up, 8 in 2024-09-20T18:39:27.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:27 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-20T18:39:27.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:27 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-20T18:39:27.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:27 smithi174 ceph-mon[49279]: Upgrade: osd.1 is safe to restart 2024-09-20T18:39:28.596 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:28 smithi012 ceph-mon[65348]: pgmap v90: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 1023 B/s rd, 48 KiB/s wr, 4 op/s; 0 B/s, 47 objects/s recovering 2024-09-20T18:39:28.596 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:28.596 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T18:39:28.597 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:39:28.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:28 smithi174 ceph-mon[49279]: pgmap v90: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 1023 B/s rd, 48 KiB/s wr, 4 op/s; 0 B/s, 47 objects/s recovering 2024-09-20T18:39:28.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:28.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-20T18:39:28.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:39:29.619 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:29 smithi012 ceph-mon[65348]: Upgrade: Updating osd.1 2024-09-20T18:39:29.619 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:29 smithi012 ceph-mon[65348]: Deploying daemon osd.1 on smithi012 2024-09-20T18:39:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:29 smithi174 ceph-mon[49279]: Upgrade: Updating osd.1 2024-09-20T18:39:29.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:29 smithi174 ceph-mon[49279]: Deploying daemon osd.1 on smithi012 2024-09-20T18:39:30.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:30 smithi012 ceph-mon[65348]: pgmap v91: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 722 B/s rd, 34 KiB/s wr, 3 op/s; 0 B/s, 33 objects/s recovering 2024-09-20T18:39:30.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:30 smithi174 ceph-mon[49279]: pgmap v91: 97 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 722 B/s rd, 34 KiB/s wr, 3 op/s; 0 B/s, 33 objects/s recovering 2024-09-20T18:39:31.710 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:31 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2024-09-20T18:39:31.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:31 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 1 pg degraded (PG_DEGRADED) 2024-09-20T18:39:31.975 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:31 smithi012 systemd[1]: Stopping Ceph osd.1 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:39:32.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:32 smithi012 ceph-mon[65348]: pgmap v92: 97 pgs: 97 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 614 B/s rd, 35 KiB/s wr, 5 op/s; 0 B/s, 33 objects/s recovering 2024-09-20T18:39:32.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:32 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-09-20T18:39:32.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:32 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:39:32.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:32 smithi012 ceph-mon[65348]: osd.1 marked itself down and dead 2024-09-20T18:39:32.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:32 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[38281]: 2024-09-20T18:39:32.047+0000 7fec1b3f0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:39:32.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:32 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[38281]: 2024-09-20T18:39:32.047+0000 7fec1b3f0700 -1 osd.1 91 *** Got signal Terminated *** 2024-09-20T18:39:32.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:32 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[38281]: 2024-09-20T18:39:32.047+0000 7fec1b3f0700 -1 osd.1 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:39:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:32 smithi174 ceph-mon[49279]: pgmap v92: 97 pgs: 97 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 614 B/s rd, 35 KiB/s wr, 5 op/s; 0 B/s, 33 objects/s recovering 2024-09-20T18:39:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:32 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg degraded) 2024-09-20T18:39:32.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:32 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:39:32.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:32 smithi174 ceph-mon[49279]: osd.1 marked itself down and dead 2024-09-20T18:39:33.225 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:32 smithi012 podman[72123]: 2024-09-20 18:39:32.880701504 +0000 UTC m=+1.030544189 container died 8d6c45aab0b4d41ddf6ad290abbf40a6ce82563eb07fa2c12a5bb2f545ee1d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, release=754, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.tags=base centos centos-stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-09-20T18:39:33.563 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:33 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:39:33.563 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:33 smithi012 ceph-mon[65348]: osdmap e92: 8 total, 7 up, 8 in 2024-09-20T18:39:33.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:33 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:39:33.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:33 smithi174 ceph-mon[49279]: osdmap e92: 8 total, 7 up, 8 in 2024-09-20T18:39:33.975 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:33 smithi012 podman[72123]: 2024-09-20 18:39:33.548888753 +0000 UTC m=+1.698731455 container cleanup 8d6c45aab0b4d41ddf6ad290abbf40a6ce82563eb07fa2c12a5bb2f545ee1d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, name=centos-stream, vcs-type=git, ceph=True, io.openshift.expose-services=, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-20T18:39:33.975 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:33 smithi012 bash[72123]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1 2024-09-20T18:39:34.300 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:34 smithi012 podman[72139]: 2024-09-20 18:39:34.010484022 +0000 UTC m=+1.127987204 container remove 8d6c45aab0b4d41ddf6ad290abbf40a6ce82563eb07fa2c12a5bb2f545ee1d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, architecture=x86_64, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870) 2024-09-20T18:39:34.674 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:34 smithi012 ceph-mon[65348]: pgmap v94: 97 pgs: 13 stale+active+clean, 84 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 5.1 KiB/s wr, 2 op/s; 0 B/s, 4 objects/s recovering 2024-09-20T18:39:34.675 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:34 smithi012 ceph-mon[65348]: osdmap e93: 8 total, 7 up, 8 in 2024-09-20T18:39:34.675 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:34 smithi012 podman[72228]: 2024-09-20 18:39:34.257728908 +0000 UTC m=+0.018128614 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:39:34.675 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:34 smithi012 podman[72228]: 2024-09-20 18:39:34.456531774 +0000 UTC m=+0.216931461 container create 041f590bc6ce88920f24fac61f7f2162464964880db7c11074cfef61c4b8f771 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T18:39:34.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:34 smithi174 ceph-mon[49279]: pgmap v94: 97 pgs: 13 stale+active+clean, 84 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 5.1 KiB/s wr, 2 op/s; 0 B/s, 4 objects/s recovering 2024-09-20T18:39:34.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:34 smithi174 ceph-mon[49279]: osdmap e93: 8 total, 7 up, 8 in 2024-09-20T18:39:34.975 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:34 smithi012 podman[72228]: 2024-09-20 18:39:34.8326521 +0000 UTC m=+0.593051798 container init 041f590bc6ce88920f24fac61f7f2162464964880db7c11074cfef61c4b8f771 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-20T18:39:34.975 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:34 smithi012 podman[72228]: 2024-09-20 18:39:34.841860412 +0000 UTC m=+0.602260101 container start 041f590bc6ce88920f24fac61f7f2162464964880db7c11074cfef61c4b8f771 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-20T18:39:35.356 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:35 smithi012 podman[72228]: 2024-09-20 18:39:35.013567471 +0000 UTC m=+0.773967167 container attach 041f590bc6ce88920f24fac61f7f2162464964880db7c11074cfef61c4b8f771 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-20T18:39:35.356 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:35 smithi012 podman[72228]: 2024-09-20 18:39:35.090900887 +0000 UTC m=+0.851300578 container died 041f590bc6ce88920f24fac61f7f2162464964880db7c11074cfef61c4b8f771 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-20T18:39:35.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:35 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 42/444 objects degraded (9.459%), 10 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:35.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:35 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 42/444 objects degraded (9.459%), 10 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:36.076 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:35 smithi012 podman[72250]: 2024-09-20 18:39:35.747365276 +0000 UTC m=+0.650240479 container remove 041f590bc6ce88920f24fac61f7f2162464964880db7c11074cfef61c4b8f771 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-20T18:39:36.076 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:35 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-20T18:39:36.077 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:35 smithi012 systemd[1]: Stopped Ceph osd.1 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:39:36.077 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:35 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.1.service: Consumed 5min 26.332s CPU time. 2024-09-20T18:39:36.333 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 systemd[1]: Starting Ceph osd.1 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:39:36.333 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 podman[72348]: 2024-09-20 18:39:36.186717668 +0000 UTC m=+0.018361307 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:39:36.586 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:36 smithi012 ceph-mon[65348]: pgmap v96: 97 pgs: 19 active+undersized, 3 stale+active+clean, 10 active+undersized+degraded, 65 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 4.9 KiB/s wr, 2 op/s; 42/444 objects degraded (9.459%); 0 B/s, 3 objects/s recovering 2024-09-20T18:39:36.586 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 podman[72348]: 2024-09-20 18:39:36.319062589 +0000 UTC m=+0.150706214 container create 39c78d1a97652b63e92d9c65ab4e7ef27e527b8c00d6f59804b9382994b0479a (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-20T18:39:36.586 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 podman[72348]: 2024-09-20 18:39:36.571522866 +0000 UTC m=+0.403166500 container init 39c78d1a97652b63e92d9c65ab4e7ef27e527b8c00d6f59804b9382994b0479a (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2) 2024-09-20T18:39:36.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:36 smithi174 ceph-mon[49279]: pgmap v96: 97 pgs: 19 active+undersized, 3 stale+active+clean, 10 active+undersized+degraded, 65 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 4.9 KiB/s wr, 2 op/s; 42/444 objects degraded (9.459%); 0 B/s, 3 objects/s recovering 2024-09-20T18:39:36.964 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 podman[72348]: 2024-09-20 18:39:36.578534535 +0000 UTC m=+0.410178174 container start 39c78d1a97652b63e92d9c65ab4e7ef27e527b8c00d6f59804b9382994b0479a (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2) 2024-09-20T18:39:36.964 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 podman[72348]: 2024-09-20 18:39:36.654992129 +0000 UTC m=+0.486635754 container attach 39c78d1a97652b63e92d9c65ab4e7ef27e527b8c00d6f59804b9382994b0479a (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True) 2024-09-20T18:39:36.965 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:36.965 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 bash[72348]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:36.965 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:36.965 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:36 smithi012 bash[72348]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:38.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:38 smithi012 ceph-mon[65348]: pgmap v97: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 4.9 KiB/s wr, 2 op/s; 52/444 objects degraded (11.712%); 0 B/s, 3 objects/s recovering 2024-09-20T18:39:38.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:38 smithi174 ceph-mon[49279]: pgmap v97: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 4.9 KiB/s wr, 2 op/s; 52/444 objects degraded (11.712%); 0 B/s, 3 objects/s recovering 2024-09-20T18:39:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-mon[65348]: pgmap v98: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 52/444 objects degraded (11.712%) 2024-09-20T18:39:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:40.975 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:39:40.975 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 bash[72348]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 bash[72348]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 bash[72348]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 bash[72348]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-20T18:39:40.976 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:40 smithi012 bash[72348]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-09-20T18:39:41.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:40 smithi174 ceph-mon[49279]: pgmap v98: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 52/444 objects degraded (11.712%) 2024-09-20T18:39:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 bash[72348]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 bash[72348]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 bash[72348]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 bash[72348]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate[72363]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 bash[72348]: --> ceph-volume lvm activate successful for osd ID: 1 2024-09-20T18:39:41.374 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 podman[72348]: 2024-09-20 18:39:41.042113458 +0000 UTC m=+4.873757089 container died 39c78d1a97652b63e92d9c65ab4e7ef27e527b8c00d6f59804b9382994b0479a (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-20T18:39:41.698 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:41 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 52/444 objects degraded (11.712%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:41.957 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 podman[72593]: 2024-09-20 18:39:41.683228444 +0000 UTC m=+0.634431691 container remove 39c78d1a97652b63e92d9c65ab4e7ef27e527b8c00d6f59804b9382994b0479a (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-activate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731) 2024-09-20T18:39:41.957 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 podman[72651]: 2024-09-20 18:39:41.797791192 +0000 UTC m=+0.017192350 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:39:41.957 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:41 smithi012 podman[72651]: 2024-09-20 18:39:41.942683328 +0000 UTC m=+0.162084473 container create 3fdcc68642636648c442c13c3dc05cd271c2abf54fd96813ed9c6c41e6f7e4ae (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-20T18:39:42.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:41 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 52/444 objects degraded (11.712%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:42.208 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:42 smithi012 podman[72651]: 2024-09-20 18:39:42.187324123 +0000 UTC m=+0.406725283 container init 3fdcc68642636648c442c13c3dc05cd271c2abf54fd96813ed9c6c41e6f7e4ae (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0) 2024-09-20T18:39:42.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:42 smithi012 podman[72651]: 2024-09-20 18:39:42.193441216 +0000 UTC m=+0.412842361 container start 3fdcc68642636648c442c13c3dc05cd271c2abf54fd96813ed9c6c41e6f7e4ae (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-20T18:39:42.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:42 smithi012 ceph-osd[72673]: -- 172.21.15.12:0/1976797655 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55905b8261e0 con 0x55905b7b4000 2024-09-20T18:39:42.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:42 smithi012 bash[72651]: 3fdcc68642636648c442c13c3dc05cd271c2abf54fd96813ed9c6c41e6f7e4ae 2024-09-20T18:39:42.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:42 smithi012 systemd[1]: Started Ceph osd.1 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:39:42.552 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:39:42.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:42 smithi012 ceph-mon[65348]: pgmap v99: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 593 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:42.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:42.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:42.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:39:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:42 smithi174 ceph-mon[49279]: pgmap v99: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 593 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:43.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:39:43.311 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (4m) 38s ago 19m 15.9M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (89s) 38s ago 19m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (82s) 78s ago 18m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (4m) 38s ago 18m 44.0M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (13m) 38s ago 13m 41.6M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (13m) 78s ago 13m 15.9M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (3m) 38s ago 20m 515M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (2m) 78s ago 18m 454M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (2m) 38s ago 20m 48.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (2m) 78s ago 18m 36.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:39:43.312 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (12m) 38s ago 12m 289M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (12m) 78s ago 12m 83.9M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (4m) 38s ago 18m 9.94M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (3m) 78s ago 18m 11.9M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (47s) 38s ago 17m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 starting - - - 1937M 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (16m) 38s ago 16m 419M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (16m) 38s ago 16m 510M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (15m) 78s ago 15m 463M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (15m) 78s ago 15m 578M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (15m) 78s ago 15m 509M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (14m) 78s ago 14m 765M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:39:43.313 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (3m) 38s ago 18m 48.7M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:43.776 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 5 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:39:43.777 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:39:43.935 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:43 smithi012 ceph-mon[65348]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:44.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:43 smithi174 ceph-mon[49279]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:44.154 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:39:44.154 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "7/23 daemons upgraded", 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:39:44.155 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:39:44.626 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 52/444 objects degraded (11.712%), 13 pgs degraded 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: osd.1 (root=default,host=smithi012) is down 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 52/444 objects degraded (11.712%), 13 pgs degraded 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-20T18:39:44.627 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-20T18:39:44.628 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-20T18:39:44.681 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:44 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[72666]: 2024-09-20T18:39:44.383+0000 7fbb0e376640 -1 Falling back to public interface 2024-09-20T18:39:44.681 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:44 smithi012 ceph-mon[65348]: pgmap v100: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 511 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:44.681 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:44 smithi012 ceph-mon[65348]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:44.681 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:44 smithi012 ceph-mon[65348]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:44.682 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:44 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/252366641' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:45.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:44 smithi174 ceph-mon[49279]: pgmap v100: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 511 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:45.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:44 smithi174 ceph-mon[49279]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:45.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:44 smithi174 ceph-mon[49279]: from='client.34238 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:45.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:44 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/252366641' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:45.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:45 smithi012 ceph-mon[65348]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:45.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:45 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/91894144' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:39:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:45 smithi174 ceph-mon[49279]: from='client.34246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:39:46.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:45 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/91894144' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:39:46.946 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:46 smithi012 ceph-mon[65348]: pgmap v101: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 440 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:47.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:46 smithi174 ceph-mon[49279]: pgmap v101: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 440 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:48.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:48 smithi012 ceph-mon[65348]: pgmap v102: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:48.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:48.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:49.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:48 smithi174 ceph-mon[49279]: pgmap v102: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:49.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:49.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:50.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:50.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:50.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:50.855 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:50 smithi012 ceph-mon[65348]: pgmap v103: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:51.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:50 smithi174 ceph-mon[49279]: pgmap v103: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:51.725 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:51 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[72666]: 2024-09-20T18:39:51.362+0000 7fbb0e376640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-20T18:39:52.725 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:52 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[72666]: 2024-09-20T18:39:52.400+0000 7fbb0e376640 -1 osd.1 91 log_to_monitors true 2024-09-20T18:39:53.117 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:52 smithi012 ceph-mon[65348]: pgmap v104: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:53.117 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:52 smithi012 ceph-mon[65348]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-20T18:39:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:52 smithi174 ceph-mon[49279]: pgmap v104: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 170 B/s rd, 426 B/s wr, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:52 smithi174 ceph-mon[49279]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-20T18:39:53.475 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[72666]: 2024-09-20T18:39:53.296+0000 7fbb02118640 -1 osd.1 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:39:53.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: osdmap e94: 8 total, 7 up, 8 in 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' 2024-09-20T18:39:53.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:39:53.977 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:53.977 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:53.977 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:53.977 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: osdmap e94: 8 total, 7 up, 8 in 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603]' entity='osd.1' 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:39:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:54.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:54.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:39:54.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T18:39:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: pgmap v106: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T18:39:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-20T18:39:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:39:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603] boot 2024-09-20T18:39:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T18:39:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T18:39:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: pgmap v106: 97 pgs: 26 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 52/444 objects degraded (11.712%) 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: osd.1 [v2:172.21.15.12:6810/1094036603,v1:172.21.15.12:6811/1094036603] boot 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: osdmap e95: 8 total, 8 up, 8 in 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:39:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:39:56.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:56 smithi012 ceph-mon[65348]: pgmap v108: 97 pgs: 19 peering, 13 active+undersized, 7 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32/444 objects degraded (7.207%) 2024-09-20T18:39:56.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:56 smithi012 ceph-mon[65348]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T18:39:56.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:56 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 32/444 objects degraded (7.207%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:56.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:56 smithi174 ceph-mon[49279]: pgmap v108: 97 pgs: 19 peering, 13 active+undersized, 7 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 32/444 objects degraded (7.207%) 2024-09-20T18:39:56.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:56 smithi174 ceph-mon[49279]: osdmap e96: 8 total, 8 up, 8 in 2024-09-20T18:39:56.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:56 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 32/444 objects degraded (7.207%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:39:58.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:58 smithi012 ceph-mon[65348]: pgmap v110: 97 pgs: 26 peering, 7 active+undersized, 6 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 31/444 objects degraded (6.982%) 2024-09-20T18:39:58.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:58 smithi174 ceph-mon[49279]: pgmap v110: 97 pgs: 26 peering, 7 active+undersized, 6 active+undersized+degraded, 58 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 31/444 objects degraded (6.982%) 2024-09-20T18:39:59.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:59 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/444 objects degraded (6.982%), 6 pgs degraded) 2024-09-20T18:39:59.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:39:59 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:39:59.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:59 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/444 objects degraded (6.982%), 6 pgs degraded) 2024-09-20T18:39:59.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:39:59 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:40:00.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:00 smithi012 ceph-mon[65348]: pgmap v111: 97 pgs: 26 peering, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 334 B/s rd, 0 op/s; 695 B/s, 0 objects/s recovering 2024-09-20T18:40:00.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:00 smithi012 ceph-mon[65348]: overall HEALTH_OK 2024-09-20T18:40:00.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:00 smithi174 ceph-mon[49279]: pgmap v111: 97 pgs: 26 peering, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 334 B/s rd, 0 op/s; 695 B/s, 0 objects/s recovering 2024-09-20T18:40:00.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:00 smithi174 ceph-mon[49279]: overall HEALTH_OK 2024-09-20T18:40:02.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:02 smithi012 ceph-mon[65348]: pgmap v112: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 532 B/s, 0 objects/s recovering 2024-09-20T18:40:02.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:02 smithi174 ceph-mon[49279]: pgmap v112: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 532 B/s, 0 objects/s recovering 2024-09-20T18:40:04.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:04 smithi012 ceph-mon[65348]: pgmap v113: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 236 B/s rd, 0 op/s; 492 B/s, 0 objects/s recovering 2024-09-20T18:40:04.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:04 smithi174 ceph-mon[49279]: pgmap v113: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 236 B/s rd, 0 op/s; 492 B/s, 0 objects/s recovering 2024-09-20T18:40:06.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:06 smithi012 ceph-mon[65348]: pgmap v114: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 426 B/s, 0 objects/s recovering 2024-09-20T18:40:06.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:06 smithi174 ceph-mon[49279]: pgmap v114: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 426 B/s, 0 objects/s recovering 2024-09-20T18:40:08.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:08 smithi012 ceph-mon[65348]: pgmap v115: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 175 B/s rd, 0 op/s; 365 B/s, 0 objects/s recovering 2024-09-20T18:40:08.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:08 smithi174 ceph-mon[49279]: pgmap v115: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 175 B/s rd, 0 op/s; 365 B/s, 0 objects/s recovering 2024-09-20T18:40:09.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T18:40:09.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:09 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T18:40:09.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:09 smithi012 ceph-mon[65348]: Upgrade: osd.2 is safe to restart 2024-09-20T18:40:09.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-20T18:40:09.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:09 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-20T18:40:09.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:09 smithi174 ceph-mon[49279]: Upgrade: osd.2 is safe to restart 2024-09-20T18:40:10.784 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:10 smithi012 ceph-mon[65348]: pgmap v116: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 355 B/s, 0 objects/s recovering 2024-09-20T18:40:10.784 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:10.784 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:10.784 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:10.784 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:10.784 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T18:40:10.785 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:40:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:10 smithi174 ceph-mon[49279]: pgmap v116: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 355 B/s, 0 objects/s recovering 2024-09-20T18:40:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-20T18:40:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:40:11.877 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:11 smithi012 ceph-mon[65348]: Upgrade: Updating osd.2 2024-09-20T18:40:11.877 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:11 smithi012 ceph-mon[65348]: Deploying daemon osd.2 on smithi012 2024-09-20T18:40:12.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:11 smithi174 ceph-mon[49279]: Upgrade: Updating osd.2 2024-09-20T18:40:12.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:11 smithi174 ceph-mon[49279]: Deploying daemon osd.2 on smithi012 2024-09-20T18:40:12.873 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:12 smithi012 ceph-mon[65348]: pgmap v117: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:13.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:12 smithi174 ceph-mon[49279]: pgmap v117: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:13.577 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:13 smithi012 systemd[1]: Stopping Ceph osd.2 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:40:13.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:13 smithi012 ceph-mon[65348]: osd.2 marked itself down and dead 2024-09-20T18:40:13.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:13 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[41946]: 2024-09-20T18:40:13.561+0000 7f7b48755700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:40:13.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:13 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[41946]: 2024-09-20T18:40:13.561+0000 7f7b48755700 -1 osd.2 96 *** Got signal Terminated *** 2024-09-20T18:40:13.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:13 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[41946]: 2024-09-20T18:40:13.561+0000 7f7b48755700 -1 osd.2 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:40:14.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:13 smithi174 ceph-mon[49279]: osd.2 marked itself down and dead 2024-09-20T18:40:14.475 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:14 smithi012 podman[74566]: 2024-09-20 18:40:14.304021499 +0000 UTC m=+0.873178418 container died 5817293a567287e7838f0dacba849f6c4394251a650a3305b22d5d457bddaae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public, vcs-type=git, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, version=8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.expose-services=, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-09-20T18:40:14.904 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:14 smithi012 ceph-mon[65348]: pgmap v118: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:14.904 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:14 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:40:14.905 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:14 smithi012 ceph-mon[65348]: osdmap e97: 8 total, 7 up, 8 in 2024-09-20T18:40:14.905 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:14 smithi012 podman[74566]: 2024-09-20 18:40:14.712771475 +0000 UTC m=+1.281928384 container cleanup 5817293a567287e7838f0dacba849f6c4394251a650a3305b22d5d457bddaae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, io.openshift.expose-services=, release=754, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-20T18:40:14.905 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:14 smithi012 bash[74566]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2 2024-09-20T18:40:15.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:14 smithi174 ceph-mon[49279]: pgmap v118: 97 pgs: 97 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:15.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:14 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:40:15.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:14 smithi174 ceph-mon[49279]: osdmap e97: 8 total, 7 up, 8 in 2024-09-20T18:40:15.074 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:40:15.225 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:14 smithi012 podman[74582]: 2024-09-20 18:40:14.889288561 +0000 UTC m=+0.581510653 container remove 5817293a567287e7838f0dacba849f6c4394251a650a3305b22d5d457bddaae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, architecture=x86_64, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, release=754) 2024-09-20T18:40:15.636 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:15 smithi012 ceph-mon[65348]: osdmap e98: 8 total, 7 up, 8 in 2024-09-20T18:40:15.636 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:15 smithi012 podman[74717]: 2024-09-20 18:40:15.269888916 +0000 UTC m=+0.017138372 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:40:15.636 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:15 smithi012 podman[74717]: 2024-09-20 18:40:15.373496015 +0000 UTC m=+0.120745455 container create df7afb24779188f66f12b9754d4958f41feb1462ab42bc808f7b8fd7eb50485b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True) 2024-09-20T18:40:15.636 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:15 smithi012 podman[74717]: 2024-09-20 18:40:15.609462415 +0000 UTC m=+0.356711855 container init df7afb24779188f66f12b9754d4958f41feb1462ab42bc808f7b8fd7eb50485b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (5m) 27s ago 19m 16.0M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (2m) 27s ago 19m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (115s) 110s ago 19m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (5m) 27s ago 19m 45.0M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (13m) 27s ago 13m 41.1M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (13m) 110s ago 13m 15.9M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (4m) 27s ago 20m 520M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (3m) 110s ago 19m 454M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (3m) 27s ago 21m 50.0M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:40:15.835 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (2m) 110s ago 18m 36.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (13m) 27s ago 13m 268M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (13m) 110s ago 13m 83.9M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (4m) 27s ago 19m 9.88M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (4m) 110s ago 18m 11.9M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (80s) 27s ago 18m 253M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (34s) 27s ago 17m 14.3M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (17m) 27s ago 17m 445M 1937M 17.2.0 e1d6a67b021e 5817293a5672 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (16m) 27s ago 16m 532M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (16m) 110s ago 16m 463M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (15m) 110s ago 15m 578M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (15m) 110s ago 15m 509M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (15m) 110s ago 15m 765M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:40:15.836 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (3m) 27s ago 18m 50.9M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:40:15.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:15 smithi012 podman[74717]: 2024-09-20 18:40:15.620745338 +0000 UTC m=+0.367994784 container start df7afb24779188f66f12b9754d4958f41feb1462ab42bc808f7b8fd7eb50485b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-20T18:40:15.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:15 smithi012 podman[74717]: 2024-09-20 18:40:15.801479024 +0000 UTC m=+0.548728479 container attach df7afb24779188f66f12b9754d4958f41feb1462ab42bc808f7b8fd7eb50485b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:40:15.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:15 smithi012 podman[74717]: 2024-09-20 18:40:15.869383515 +0000 UTC m=+0.616632968 container died df7afb24779188f66f12b9754d4958f41feb1462ab42bc808f7b8fd7eb50485b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:40:16.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:15 smithi174 ceph-mon[49279]: osdmap e98: 8 total, 7 up, 8 in 2024-09-20T18:40:16.288 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:16.289 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:40:16.290 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:40:16.290 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:16.290 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:40:16.290 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T18:40:16.290 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 6 2024-09-20T18:40:16.290 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:40:16.290 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:40:16.681 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "8/23 daemons upgraded", 2024-09-20T18:40:16.682 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:40:16.682 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:40:16.682 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:40:16.831 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:16 smithi012 podman[74766]: 2024-09-20 18:40:16.497067737 +0000 UTC m=+0.621449291 container remove df7afb24779188f66f12b9754d4958f41feb1462ab42bc808f7b8fd7eb50485b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-20T18:40:16.831 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:16 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-20T18:40:16.831 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:16 smithi012 systemd[1]: Stopped Ceph osd.2 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:40:16.831 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:16 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.2.service: Consumed 4min 33.464s CPU time. 2024-09-20T18:40:16.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:16 smithi012 ceph-mon[65348]: pgmap v121: 97 pgs: 8 active+undersized, 3 stale+active+clean, 9 active+undersized+degraded, 77 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 32/444 objects degraded (7.207%) 2024-09-20T18:40:16.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:16 smithi012 ceph-mon[65348]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:16.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:16 smithi012 ceph-mon[65348]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:16.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:16 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 32/444 objects degraded (7.207%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T18:40:16.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:16 smithi012 ceph-mon[65348]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:16.832 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:16 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2569626563' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:17.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:16 smithi174 ceph-mon[49279]: pgmap v121: 97 pgs: 8 active+undersized, 3 stale+active+clean, 9 active+undersized+degraded, 77 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 32/444 objects degraded (7.207%) 2024-09-20T18:40:17.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:16 smithi174 ceph-mon[49279]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:17.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:16 smithi174 ceph-mon[49279]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:17.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:16 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 32/444 objects degraded (7.207%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T18:40:17.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:16 smithi174 ceph-mon[49279]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:17.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:16 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2569626563' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:17.148 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:16 smithi012 systemd[1]: Starting Ceph osd.2 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:40:17.148 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 podman[74931]: 2024-09-20 18:40:16.953935968 +0000 UTC m=+0.020227838 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:40:17.148 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 podman[74931]: 2024-09-20 18:40:17.057116868 +0000 UTC m=+0.123408731 container create 93d6a17ad4eb49378668fe4555480906e1846e08ba87b39b00665b5d90048720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:40:17.150 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 32/444 objects degraded (7.207%), 9 pgs degraded 2024-09-20T18:40:17.150 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T18:40:17.150 INFO:teuthology.orchestra.run.smithi012.stdout: osd.2 (root=default,host=smithi012) is down 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 32/444 objects degraded (7.207%), 9 pgs degraded 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-20T18:40:17.151 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-20T18:40:17.433 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 podman[74931]: 2024-09-20 18:40:17.326461278 +0000 UTC m=+0.392753152 container init 93d6a17ad4eb49378668fe4555480906e1846e08ba87b39b00665b5d90048720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:40:17.433 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 podman[74931]: 2024-09-20 18:40:17.334030553 +0000 UTC m=+0.400322411 container start 93d6a17ad4eb49378668fe4555480906e1846e08ba87b39b00665b5d90048720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:40:17.433 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 podman[74931]: 2024-09-20 18:40:17.418094703 +0000 UTC m=+0.484386586 container attach 93d6a17ad4eb49378668fe4555480906e1846e08ba87b39b00665b5d90048720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-20T18:40:17.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:17 smithi012 ceph-mon[65348]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:17.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:17 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/3518149325' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:40:17.725 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:17.725 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 bash[74931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:17.725 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:17.726 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:17 smithi012 bash[74931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:18.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:17 smithi174 ceph-mon[49279]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:18.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:17 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/3518149325' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:40:18.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:18 smithi012 ceph-mon[65348]: pgmap v122: 97 pgs: 10 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 75 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 38/444 objects degraded (8.559%) 2024-09-20T18:40:19.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:18 smithi174 ceph-mon[49279]: pgmap v122: 97 pgs: 10 active+undersized, 2 stale+active+clean, 10 active+undersized+degraded, 75 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 38/444 objects degraded (8.559%) 2024-09-20T18:40:20.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:20 smithi012 ceph-mon[65348]: pgmap v123: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:21.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:20 smithi174 ceph-mon[49279]: pgmap v123: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-20T18:40:21.643 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-09-20T18:40:21.922 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:21 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 45/444 objects degraded (10.135%), 11 pgs degraded (PG_DEGRADED) 2024-09-20T18:40:21.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 45/444 objects degraded (10.135%), 11 pgs degraded (PG_DEGRADED) 2024-09-20T18:40:21.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-20T18:40:21.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-20T18:40:21.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate[74948]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 bash[74931]: --> ceph-volume lvm activate successful for osd ID: 2 2024-09-20T18:40:21.976 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:21 smithi012 podman[74931]: 2024-09-20 18:40:21.76650804 +0000 UTC m=+4.832799920 container died 93d6a17ad4eb49378668fe4555480906e1846e08ba87b39b00665b5d90048720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-20T18:40:22.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:22 smithi012 ceph-mon[65348]: pgmap v124: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:22.725 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 podman[75167]: 2024-09-20 18:40:22.38761601 +0000 UTC m=+0.614465468 container remove 93d6a17ad4eb49378668fe4555480906e1846e08ba87b39b00665b5d90048720 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-20T18:40:22.725 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 podman[75226]: 2024-09-20 18:40:22.505543266 +0000 UTC m=+0.016919935 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:40:22.726 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 podman[75226]: 2024-09-20 18:40:22.613717325 +0000 UTC m=+0.125093993 container create 7a10de3220f0d1e4b414ed577910162c73e38cabed1527fb666636eb078bac14 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True) 2024-09-20T18:40:23.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:22 smithi174 ceph-mon[49279]: pgmap v124: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:23.225 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 podman[75226]: 2024-09-20 18:40:22.874397883 +0000 UTC m=+0.385774552 container init 7a10de3220f0d1e4b414ed577910162c73e38cabed1527fb666636eb078bac14 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:40:23.225 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 podman[75226]: 2024-09-20 18:40:22.878813084 +0000 UTC m=+0.390189754 container start 7a10de3220f0d1e4b414ed577910162c73e38cabed1527fb666636eb078bac14 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-20T18:40:23.225 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 ceph-osd[75245]: -- 172.21.15.12:0/1727509138 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555b0ed9c1e0 con 0x555b0ed28000 2024-09-20T18:40:23.225 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 bash[75226]: 7a10de3220f0d1e4b414ed577910162c73e38cabed1527fb666636eb078bac14 2024-09-20T18:40:23.225 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:22 smithi012 systemd[1]: Started Ceph osd.2 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:40:24.366 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:24 smithi012 ceph-mon[65348]: pgmap v125: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 219 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:24.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:24.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:24.367 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:40:24.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:24 smithi174 ceph-mon[49279]: pgmap v125: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 219 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:24.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:24.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:24.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:40:25.475 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:25 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[75241]: 2024-09-20T18:40:25.044+0000 7f7eda048640 -1 Falling back to public interface 2024-09-20T18:40:25.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:25.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:26.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:26.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:26.964 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:26 smithi012 ceph-mon[65348]: pgmap v126: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 396 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:27.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:26 smithi174 ceph-mon[49279]: pgmap v126: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 396 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:28.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:28 smithi012 ceph-mon[65348]: pgmap v127: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:28.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:28.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:29.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:28 smithi174 ceph-mon[49279]: pgmap v127: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:30.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:30 smithi012 ceph-mon[65348]: pgmap v128: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:30.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:30.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:30.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:30 smithi174 ceph-mon[49279]: pgmap v128: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:30.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:30.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:32.188 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:31 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[75241]: 2024-09-20T18:40:31.788+0000 7f7eda048640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-20T18:40:32.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:32 smithi012 ceph-mon[65348]: pgmap v129: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:32.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:32 smithi174 ceph-mon[49279]: pgmap v129: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:33.197 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:32 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[75241]: 2024-09-20T18:40:32.846+0000 7f7eda048640 -1 osd.2 96 log_to_monitors true 2024-09-20T18:40:33.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:33 smithi012 ceph-mon[65348]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-20T18:40:33.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:33 smithi174 ceph-mon[49279]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-20T18:40:33.975 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:40:33 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[75241]: 2024-09-20T18:40:33.689+0000 7f7ecddea640 -1 osd.2 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:40:34.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: pgmap v130: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:34.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: osdmap e99: 8 total, 7 up, 8 in 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:34.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: pgmap v130: 97 pgs: 15 active+undersized, 11 active+undersized+degraded, 71 active+clean; 246 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 45/444 objects degraded (10.135%) 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: osdmap e99: 8 total, 7 up, 8 in 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:40:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:34.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696]' entity='osd.2' 2024-09-20T18:40:34.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:40:34.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:34.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:34.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:34.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T18:40:35.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:35 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T18:40:35.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:35 smithi012 ceph-mon[65348]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-20T18:40:35.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:35 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:40:35.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:35 smithi012 ceph-mon[65348]: osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696] boot 2024-09-20T18:40:35.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:35 smithi012 ceph-mon[65348]: osdmap e100: 8 total, 8 up, 8 in 2024-09-20T18:40:35.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T18:40:35.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:35 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T18:40:35.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:35 smithi174 ceph-mon[49279]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-20T18:40:35.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:35 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:40:35.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:35 smithi174 ceph-mon[49279]: osd.2 [v2:172.21.15.12:6818/1029233696,v1:172.21.15.12:6819/1029233696] boot 2024-09-20T18:40:35.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:35 smithi174 ceph-mon[49279]: osdmap e100: 8 total, 8 up, 8 in 2024-09-20T18:40:35.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-20T18:40:36.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:36 smithi012 ceph-mon[65348]: pgmap v133: 97 pgs: 13 peering, 8 active+undersized, 5 active+undersized+degraded, 71 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 24/444 objects degraded (5.405%) 2024-09-20T18:40:36.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:36 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 24/444 objects degraded (5.405%), 5 pgs degraded (PG_DEGRADED) 2024-09-20T18:40:36.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:36 smithi012 ceph-mon[65348]: osdmap e101: 8 total, 8 up, 8 in 2024-09-20T18:40:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:36 smithi174 ceph-mon[49279]: pgmap v133: 97 pgs: 13 peering, 8 active+undersized, 5 active+undersized+degraded, 71 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 24/444 objects degraded (5.405%) 2024-09-20T18:40:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:36 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 24/444 objects degraded (5.405%), 5 pgs degraded (PG_DEGRADED) 2024-09-20T18:40:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:36 smithi174 ceph-mon[49279]: osdmap e101: 8 total, 8 up, 8 in 2024-09-20T18:40:38.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:38 smithi012 ceph-mon[65348]: pgmap v135: 97 pgs: 15 peering, 6 active+undersized, 5 active+undersized+degraded, 71 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 24/444 objects degraded (5.405%) 2024-09-20T18:40:38.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:38 smithi174 ceph-mon[49279]: pgmap v135: 97 pgs: 15 peering, 6 active+undersized, 5 active+undersized+degraded, 71 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 24/444 objects degraded (5.405%) 2024-09-20T18:40:39.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:39 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/444 objects degraded (5.405%), 5 pgs degraded) 2024-09-20T18:40:39.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:39 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:40:39.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:39 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/444 objects degraded (5.405%), 5 pgs degraded) 2024-09-20T18:40:39.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:39 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:40:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:40 smithi012 ceph-mon[65348]: pgmap v136: 97 pgs: 15 peering, 82 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:40:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:40 smithi174 ceph-mon[49279]: pgmap v136: 97 pgs: 15 peering, 82 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:40:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:42.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:42 smithi012 ceph-mon[65348]: pgmap v137: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 264 B/s rd, 0 op/s 2024-09-20T18:40:43.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:42 smithi174 ceph-mon[49279]: pgmap v137: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 264 B/s rd, 0 op/s 2024-09-20T18:40:44.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:44 smithi012 ceph-mon[65348]: pgmap v138: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-09-20T18:40:45.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:44 smithi174 ceph-mon[49279]: pgmap v138: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 247 B/s rd, 0 op/s 2024-09-20T18:40:46.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:46 smithi012 ceph-mon[65348]: pgmap v139: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:40:47.020 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:46 smithi174 ceph-mon[49279]: pgmap v139: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:40:47.534 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:40:48.251 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:40:48.251 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (5m) 19s ago 20m 16.0M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:40:48.251 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (2m) 19s ago 20m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:40:48.251 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (2m) 2m ago 19m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:40:48.251 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (5m) 19s ago 20m 45.6M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:40:48.251 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (14m) 19s ago 14m 41.1M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:40:48.251 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (14m) 2m ago 14m 15.9M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (4m) 19s ago 21m 521M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (4m) 2m ago 19m 454M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (3m) 19s ago 21m 51.4M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (3m) 2m ago 19m 36.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (13m) 19s ago 13m 265M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (13m) 2m ago 13m 83.9M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (5m) 19s ago 19m 9.87M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (5m) 2m ago 19m 11.9M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (112s) 19s ago 18m 253M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (66s) 19s ago 18m 209M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (25s) 19s ago 17m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:40:48.252 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (17m) 19s ago 17m 533M 1937M 17.2.0 e1d6a67b021e d1d228c6895b 2024-09-20T18:40:48.253 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (16m) 2m ago 16m 463M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:40:48.253 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (16m) 2m ago 16m 578M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:40:48.253 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (16m) 2m ago 16m 509M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:40:48.253 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (15m) 2m ago 15m 765M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:40:48.253 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (4m) 19s ago 19m 51.0M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:40:48.695 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T18:40:48.696 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 3 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 7 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:40:48.697 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:40:48.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:48 smithi012 ceph-mon[65348]: pgmap v140: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-09-20T18:40:48.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:48 smithi012 ceph-mon[65348]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:48.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:48 smithi012 ceph-mon[65348]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:49.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:48 smithi174 ceph-mon[49279]: pgmap v140: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 181 B/s rd, 0 op/s 2024-09-20T18:40:49.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:48 smithi174 ceph-mon[49279]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:49.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:48 smithi174 ceph-mon[49279]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:49.064 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:40:49.064 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "9/23 daemons upgraded", 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:40:49.065 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:40:49.531 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:40:49.702 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:49 smithi012 ceph-mon[65348]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:49.702 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:49 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/3963171337' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:49.702 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T18:40:49.702 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:49 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/3675375443' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:40:50.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:49 smithi174 ceph-mon[49279]: from='client.34292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:50.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:49 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/3963171337' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:40:50.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-20T18:40:50.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:49 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/3675375443' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:40:50.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:50 smithi012 ceph-mon[65348]: pgmap v141: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:50.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:50 smithi012 ceph-mon[65348]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:50.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:50 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T18:40:50.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:50 smithi012 ceph-mon[65348]: Upgrade: osd.3 is safe to restart 2024-09-20T18:40:50.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:50 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:51.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:50 smithi174 ceph-mon[49279]: pgmap v141: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:51.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:50 smithi174 ceph-mon[49279]: from='client.34300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:40:51.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:50 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-20T18:40:51.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:50 smithi174 ceph-mon[49279]: Upgrade: osd.3 is safe to restart 2024-09-20T18:40:51.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:50 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:52.166 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:51 smithi012 ceph-mon[65348]: Upgrade: Updating osd.3 2024-09-20T18:40:52.166 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:51 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:52.167 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:51 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T18:40:52.167 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:51 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:40:52.167 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:51 smithi012 ceph-mon[65348]: Deploying daemon osd.3 on smithi012 2024-09-20T18:40:52.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:51 smithi174 ceph-mon[49279]: Upgrade: Updating osd.3 2024-09-20T18:40:52.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:51 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:40:52.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:51 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-20T18:40:52.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:51 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:40:52.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:51 smithi174 ceph-mon[49279]: Deploying daemon osd.3 on smithi012 2024-09-20T18:40:53.073 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:52 smithi012 ceph-mon[65348]: pgmap v142: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:52 smithi174 ceph-mon[49279]: pgmap v142: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:54.012 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:53 smithi012 systemd[1]: Stopping Ceph osd.3 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:40:54.475 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[45175]: 2024-09-20T18:40:53.996+0000 7fdb754ab700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:40:54.476 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[45175]: 2024-09-20T18:40:53.996+0000 7fdb754ab700 -1 osd.3 101 *** Got signal Terminated *** 2024-09-20T18:40:54.476 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:53 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[45175]: 2024-09-20T18:40:53.996+0000 7fdb754ab700 -1 osd.3 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:40:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:54 smithi012 ceph-mon[65348]: pgmap v143: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:54 smithi012 ceph-mon[65348]: osd.3 marked itself down and dead 2024-09-20T18:40:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:54 smithi174 ceph-mon[49279]: pgmap v143: 97 pgs: 97 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:40:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:54 smithi174 ceph-mon[49279]: osd.3 marked itself down and dead 2024-09-20T18:40:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:40:55.725 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:55 smithi012 podman[77152]: 2024-09-20 18:40:55.437373041 +0000 UTC m=+1.552431634 container died d1d228c6895b7ff283297405f243ca457d315d15fe71ece8dcf92db10f9b648e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, architecture=x86_64, version=8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, name=centos-stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container) 2024-09-20T18:40:56.079 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:55 smithi012 podman[77152]: 2024-09-20 18:40:55.812643273 +0000 UTC m=+1.927701861 container cleanup d1d228c6895b7ff283297405f243ca457d315d15fe71ece8dcf92db10f9b648e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.expose-services=, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, RELEASE=HEAD, version=8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-type=git) 2024-09-20T18:40:56.079 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:55 smithi012 bash[77152]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3 2024-09-20T18:40:56.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:55 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:40:56.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:55 smithi012 ceph-mon[65348]: osdmap e102: 8 total, 7 up, 8 in 2024-09-20T18:40:56.080 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:55 smithi012 ceph-mon[65348]: osdmap e103: 8 total, 7 up, 8 in 2024-09-20T18:40:56.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:55 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:40:56.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:55 smithi174 ceph-mon[49279]: osdmap e102: 8 total, 7 up, 8 in 2024-09-20T18:40:56.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:55 smithi174 ceph-mon[49279]: osdmap e103: 8 total, 7 up, 8 in 2024-09-20T18:40:56.400 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:56 smithi012 podman[77167]: 2024-09-20 18:40:56.064166476 +0000 UTC m=+0.624395455 container remove d1d228c6895b7ff283297405f243ca457d315d15fe71ece8dcf92db10f9b648e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, RELEASE=HEAD, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, ceph=True, architecture=x86_64, maintainer=Guillaume Abrioux , version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.openshift.tags=base centos centos-stream) 2024-09-20T18:40:56.674 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:56 smithi012 podman[77257]: 2024-09-20 18:40:56.28852949 +0000 UTC m=+0.016879937 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:40:56.674 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:56 smithi012 podman[77257]: 2024-09-20 18:40:56.406765577 +0000 UTC m=+0.135116066 container create 22dfe54dfec3666f3c469a6235f07d858079d5eeb978d52cc5cb2d4f95ad1e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-20T18:40:56.674 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:56 smithi012 podman[77257]: 2024-09-20 18:40:56.659399504 +0000 UTC m=+0.387749951 container init 22dfe54dfec3666f3c469a6235f07d858079d5eeb978d52cc5cb2d4f95ad1e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-20T18:40:56.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:56 smithi012 ceph-mon[65348]: pgmap v145: 97 pgs: 10 peering, 87 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:40:56.976 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:56 smithi012 podman[77257]: 2024-09-20 18:40:56.663074044 +0000 UTC m=+0.391424490 container start 22dfe54dfec3666f3c469a6235f07d858079d5eeb978d52cc5cb2d4f95ad1e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:40:56.976 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:56 smithi012 podman[77257]: 2024-09-20 18:40:56.742659868 +0000 UTC m=+0.471010321 container attach 22dfe54dfec3666f3c469a6235f07d858079d5eeb978d52cc5cb2d4f95ad1e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-20T18:40:56.976 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:56 smithi012 podman[77257]: 2024-09-20 18:40:56.86165252 +0000 UTC m=+0.590002966 container died 22dfe54dfec3666f3c469a6235f07d858079d5eeb978d52cc5cb2d4f95ad1e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True) 2024-09-20T18:40:57.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:56 smithi174 ceph-mon[49279]: pgmap v145: 97 pgs: 10 peering, 87 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:40:57.436 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:57 smithi012 podman[77281]: 2024-09-20 18:40:57.404777482 +0000 UTC m=+0.535227549 container remove 22dfe54dfec3666f3c469a6235f07d858079d5eeb978d52cc5cb2d4f95ad1e2e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:40:57.725 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:57 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-20T18:40:57.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:57 smithi012 systemd[1]: Stopped Ceph osd.3 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:40:57.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:57 smithi012 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.3.service: Consumed 5min 44.011s CPU time. 2024-09-20T18:40:57.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:57 smithi012 systemd[1]: Starting Ceph osd.3 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:40:58.182 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:57 smithi012 ceph-mon[65348]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-09-20T18:40:58.182 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:57 smithi012 podman[77379]: 2024-09-20 18:40:57.763984433 +0000 UTC m=+0.016764052 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:40:58.183 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:57 smithi012 podman[77379]: 2024-09-20 18:40:57.88089605 +0000 UTC m=+0.133675711 container create 1d7b364fc47a6c1d975f213a8f83c0df7b1a7688f78de1ac41f3607064b0e639 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T18:40:58.183 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:58 smithi012 podman[77379]: 2024-09-20 18:40:58.083702064 +0000 UTC m=+0.336481683 container init 1d7b364fc47a6c1d975f213a8f83c0df7b1a7688f78de1ac41f3607064b0e639 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-20T18:40:58.183 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:58 smithi012 podman[77379]: 2024-09-20 18:40:58.087364511 +0000 UTC m=+0.340144138 container start 1d7b364fc47a6c1d975f213a8f83c0df7b1a7688f78de1ac41f3607064b0e639 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-20T18:40:58.183 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:58 smithi012 podman[77379]: 2024-09-20 18:40:58.167202308 +0000 UTC m=+0.419981942 container attach 1d7b364fc47a6c1d975f213a8f83c0df7b1a7688f78de1ac41f3607064b0e639 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-20T18:40:58.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:57 smithi174 ceph-mon[49279]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-09-20T18:40:58.475 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:58 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:58.475 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:58 smithi012 bash[77379]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:58.475 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:58 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:58.475 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:40:58 smithi012 bash[77379]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:40:59.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:58 smithi012 ceph-mon[65348]: pgmap v147: 97 pgs: 13 stale+active+clean, 19 peering, 65 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:40:59.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:58 smithi174 ceph-mon[49279]: pgmap v147: 97 pgs: 13 stale+active+clean, 19 peering, 65 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:41:00.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:59 smithi174 ceph-mon[49279]: pgmap v148: 97 pgs: 6 active+undersized, 9 stale+active+clean, 19 peering, 7 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 28/444 objects degraded (6.306%) 2024-09-20T18:41:00.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:40:59 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 28/444 objects degraded (6.306%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:00.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:59 smithi012 ceph-mon[65348]: pgmap v148: 97 pgs: 6 active+undersized, 9 stale+active+clean, 19 peering, 7 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 28/444 objects degraded (6.306%) 2024-09-20T18:41:00.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:40:59 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 28/444 objects degraded (6.306%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:01.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-mon[65348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-20T18:41:01.427 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-09-20T18:41:01.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:01 smithi174 ceph-mon[49279]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-09-20T18:41:01.725 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-20T18:41:01.725 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-20T18:41:01.725 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate[77394]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 bash[77379]: --> ceph-volume lvm activate successful for osd ID: 3 2024-09-20T18:41:01.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:01 smithi012 podman[77379]: 2024-09-20 18:41:01.557042984 +0000 UTC m=+3.809822603 container died 1d7b364fc47a6c1d975f213a8f83c0df7b1a7688f78de1ac41f3607064b0e639 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-20T18:41:02.177 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:02 smithi012 ceph-mon[65348]: pgmap v149: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:02.475 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 podman[77625]: 2024-09-20 18:41:02.162227457 +0000 UTC m=+0.598413353 container remove 1d7b364fc47a6c1d975f213a8f83c0df7b1a7688f78de1ac41f3607064b0e639 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-activate, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-20T18:41:02.475 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 podman[77685]: 2024-09-20 18:41:02.287228053 +0000 UTC m=+0.017315162 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:41:02.476 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 podman[77685]: 2024-09-20 18:41:02.396480731 +0000 UTC m=+0.126567813 container create 8b4734cb5150e05c432e1debddfaf8d4053dae2f70c07eefdd5cac76caa09893 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-20T18:41:02.520 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:02 smithi174 ceph-mon[49279]: pgmap v149: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:02.975 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 podman[77685]: 2024-09-20 18:41:02.598757607 +0000 UTC m=+0.328844710 container init 8b4734cb5150e05c432e1debddfaf8d4053dae2f70c07eefdd5cac76caa09893 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=) 2024-09-20T18:41:02.975 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 podman[77685]: 2024-09-20 18:41:02.602227776 +0000 UTC m=+0.332314863 container start 8b4734cb5150e05c432e1debddfaf8d4053dae2f70c07eefdd5cac76caa09893 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:41:02.976 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 ceph-osd[77703]: -- 172.21.15.12:0/1416911342 <== mon.0 v2:172.21.15.12:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560c705881e0 con 0x560c70584000 2024-09-20T18:41:02.976 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 bash[77685]: 8b4734cb5150e05c432e1debddfaf8d4053dae2f70c07eefdd5cac76caa09893 2024-09-20T18:41:02.976 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:02 smithi012 systemd[1]: Started Ceph osd.3 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:41:04.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:04.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:04.079 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:04.780 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:04 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:41:04.487+0000 7f109ef82640 -1 Falling back to public interface 2024-09-20T18:41:05.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:04 smithi012 ceph-mon[65348]: pgmap v150: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:05.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:04 smithi174 ceph-mon[49279]: pgmap v150: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:07.178 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:06 smithi012 ceph-mon[65348]: pgmap v151: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:07.178 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:06 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 60/444 objects degraded (13.514%), 14 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:07.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:06 smithi174 ceph-mon[49279]: pgmap v151: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:07.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:06 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 60/444 objects degraded (13.514%), 14 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:09.187 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:08 smithi012 ceph-mon[65348]: pgmap v152: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:09.188 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:08 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:09.188 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:08 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:09.188 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:08 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:09.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:08 smithi174 ceph-mon[49279]: pgmap v152: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:09.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:08 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:09.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:08 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:09.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:08 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:10.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:10.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:10.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:10.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:10.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:10.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:11.065 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:10 smithi012 ceph-mon[65348]: pgmap v153: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:11.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:10 smithi174 ceph-mon[49279]: pgmap v153: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 60/444 objects degraded (13.514%) 2024-09-20T18:41:11.426 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:11 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:41:11.047+0000 7f109ef82640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-20T18:41:12.725 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:41:12.308+0000 7f109ef82640 -1 osd.3 101 log_to_monitors true 2024-09-20T18:41:13.225 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:41:12.810+0000 7f1096d2c640 -1 osd.3 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: pgmap v154: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 60/444 objects degraded (13.514%) 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: from='osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: pgmap v154: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 60/444 objects degraded (13.514%) 2024-09-20T18:41:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: from='osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-20T18:41:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:13.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:13.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:41:13.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:41:13.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:13.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: osdmap e104: 8 total, 7 up, 8 in 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T18:41:14.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:13 smithi012 ceph-mon[65348]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: osdmap e104: 8 total, 7 up, 8 in 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi012", "root=default"]} : dispatch 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T18:41:14.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:13 smithi174 ceph-mon[49279]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T18:41:15.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:14 smithi012 ceph-mon[65348]: pgmap v156: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 60/444 objects degraded (13.514%) 2024-09-20T18:41:15.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:14 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:41:15.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:14 smithi012 ceph-mon[65348]: osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749] boot 2024-09-20T18:41:15.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:14 smithi012 ceph-mon[65348]: osdmap e105: 8 total, 8 up, 8 in 2024-09-20T18:41:15.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:14 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T18:41:15.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:14 smithi174 ceph-mon[49279]: pgmap v156: 97 pgs: 27 active+undersized, 14 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 60/444 objects degraded (13.514%) 2024-09-20T18:41:15.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:14 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:41:15.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:14 smithi174 ceph-mon[49279]: osd.3 [v2:172.21.15.12:6826/1631286749,v1:172.21.15.12:6827/1631286749] boot 2024-09-20T18:41:15.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:14 smithi174 ceph-mon[49279]: osdmap e105: 8 total, 8 up, 8 in 2024-09-20T18:41:15.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:14 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-20T18:41:16.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:15 smithi012 ceph-mon[65348]: osdmap e106: 8 total, 8 up, 8 in 2024-09-20T18:41:16.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:15 smithi174 ceph-mon[49279]: osdmap e106: 8 total, 8 up, 8 in 2024-09-20T18:41:17.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:16 smithi012 ceph-mon[65348]: pgmap v159: 97 pgs: 19 active+undersized, 12 active+undersized+degraded, 66 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 48/444 objects degraded (10.811%) 2024-09-20T18:41:17.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:16 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 48/444 objects degraded (10.811%), 12 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:17.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:16 smithi174 ceph-mon[49279]: pgmap v159: 97 pgs: 19 active+undersized, 12 active+undersized+degraded, 66 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 48/444 objects degraded (10.811%) 2024-09-20T18:41:17.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:16 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 48/444 objects degraded (10.811%), 12 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:19.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:18 smithi012 ceph-mon[65348]: pgmap v160: 97 pgs: 14 active+undersized, 11 active+undersized+degraded, 72 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 40/444 objects degraded (9.009%) 2024-09-20T18:41:19.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:18 smithi174 ceph-mon[49279]: pgmap v160: 97 pgs: 14 active+undersized, 11 active+undersized+degraded, 72 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 40/444 objects degraded (9.009%) 2024-09-20T18:41:19.929 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:41:20.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:19 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/444 objects degraded (9.009%), 11 pgs degraded) 2024-09-20T18:41:20.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:19 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:41:20.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:19 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 40/444 objects degraded (9.009%), 11 pgs degraded) 2024-09-20T18:41:20.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:19 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:41:20.640 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:41:20.640 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (6m) 12s ago 20m 16.1M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:41:20.640 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (3m) 12s ago 20m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:41:20.640 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (3m) 2m ago 20m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:41:20.640 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (6m) 12s ago 20m 45.4M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:41:20.640 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (14m) 12s ago 14m 41.1M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:41:20.640 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (14m) 2m ago 14m 15.9M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (5m) 12s ago 22m 522M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (4m) 2m ago 20m 454M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (4m) 12s ago 22m 53.1M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (3m) 2m ago 20m 36.8M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (14m) 12s ago 14m 265M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (14m) 2m ago 14m 83.9M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (6m) 12s ago 20m 9.92M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (5m) 2m ago 19m 11.9M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (2m) 12s ago 19m 252M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (98s) 12s ago 18m 209M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (58s) 12s ago 18m 193M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (18s) 12s ago 17m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (17m) 2m ago 17m 463M 2704M 17.2.0 e1d6a67b021e e0ddd37493ba 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (17m) 2m ago 17m 578M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (16m) 2m ago 16m 509M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:41:20.641 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (16m) 2m ago 16m 765M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:41:20.642 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (4m) 12s ago 19m 55.3M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:41:20.890 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:20 smithi012 ceph-mon[65348]: pgmap v161: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 332 B/s rd, 0 op/s 2024-09-20T18:41:20.890 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:20 smithi012 ceph-mon[65348]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:21.077 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:41:21.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:41:21.077 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:41:21.077 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:21.077 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 4 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 8 2024-09-20T18:41:21.078 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:41:21.079 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:41:21.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:20 smithi174 ceph-mon[49279]: pgmap v161: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 332 B/s rd, 0 op/s 2024-09-20T18:41:21.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:20 smithi174 ceph-mon[49279]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:21.440 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:41:21.440 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:41:21.440 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "10/23 daemons upgraded", 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:41:21.441 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:41:21.875 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:41:22.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:21 smithi012 ceph-mon[65348]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:22.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:21 smithi012 ceph-mon[65348]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:22.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:21 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/4259556831' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:22.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:21 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2466261459' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:41:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:21 smithi174 ceph-mon[49279]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:21 smithi174 ceph-mon[49279]: from='client.34320 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:21 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/4259556831' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:21 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2466261459' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:41:23.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:22 smithi012 ceph-mon[65348]: pgmap v162: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-20T18:41:23.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:22 smithi012 ceph-mon[65348]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:23.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:22 smithi174 ceph-mon[49279]: pgmap v162: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-09-20T18:41:23.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:22 smithi174 ceph-mon[49279]: from='client.34328 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:25.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:24 smithi012 ceph-mon[65348]: pgmap v163: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-09-20T18:41:25.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:25.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:24 smithi174 ceph-mon[49279]: pgmap v163: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 223 B/s rd, 0 op/s 2024-09-20T18:41:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:27.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:26 smithi012 ceph-mon[65348]: pgmap v164: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-09-20T18:41:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:26 smithi174 ceph-mon[49279]: pgmap v164: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 201 B/s rd, 0 op/s 2024-09-20T18:41:28.213 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:27 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T18:41:28.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:27 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-20T18:41:29.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:28 smithi012 ceph-mon[65348]: pgmap v165: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:41:29.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:28 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T18:41:29.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:28 smithi012 ceph-mon[65348]: Upgrade: osd.4 is safe to restart 2024-09-20T18:41:29.231 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:28 smithi174 ceph-mon[49279]: pgmap v165: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:41:29.231 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:28 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-20T18:41:29.232 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:28 smithi174 ceph-mon[49279]: Upgrade: osd.4 is safe to restart 2024-09-20T18:41:30.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:30 smithi174 ceph-mon[49279]: pgmap v166: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:41:30.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:30 smithi174 ceph-mon[49279]: Upgrade: Updating osd.4 2024-09-20T18:41:30.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:30.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T18:41:30.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:41:30.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:30 smithi174 ceph-mon[49279]: Deploying daemon osd.4 on smithi174 2024-09-20T18:41:30.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:30 smithi012 ceph-mon[65348]: pgmap v166: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:41:30.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:30 smithi012 ceph-mon[65348]: Upgrade: Updating osd.4 2024-09-20T18:41:30.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:30.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-20T18:41:30.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:41:30.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:30 smithi012 ceph-mon[65348]: Deploying daemon osd.4 on smithi174 2024-09-20T18:41:32.335 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:32 smithi174 ceph-mon[49279]: pgmap v167: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:41:32.335 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:32 smithi174 systemd[1]: Stopping Ceph osd.4 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:41:32.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:32 smithi012 ceph-mon[65348]: pgmap v167: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:41:32.770 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[29913]: 2024-09-20T18:41:32.318+0000 7fde46454700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:41:32.771 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[29913]: 2024-09-20T18:41:32.318+0000 7fde46454700 -1 osd.4 106 *** Got signal Terminated *** 2024-09-20T18:41:32.771 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[29913]: 2024-09-20T18:41:32.318+0000 7fde46454700 -1 osd.4 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:41:33.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:33 smithi012 ceph-mon[65348]: osd.4 marked itself down and dead 2024-09-20T18:41:33.770 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:33 smithi174 ceph-mon[49279]: osd.4 marked itself down and dead 2024-09-20T18:41:34.150 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:33 smithi174 podman[52692]: 2024-09-20 18:41:33.793428308 +0000 UTC m=+1.576554378 container died e0ddd37493babbb36273a9d8972ce8dfa0363618f6efbb86ac18bc1ed16df47f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, distribution-scope=public, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_CLEAN=True, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, architecture=x86_64, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T18:41:34.151 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:34 smithi174 podman[52692]: 2024-09-20 18:41:34.135660361 +0000 UTC m=+1.918786421 container cleanup e0ddd37493babbb36273a9d8972ce8dfa0363618f6efbb86ac18bc1ed16df47f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.buildah.version=1.19.8, release=754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, maintainer=Guillaume Abrioux , vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=) 2024-09-20T18:41:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:34 smithi174 ceph-mon[49279]: pgmap v168: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:41:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:34 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:41:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:34 smithi174 ceph-mon[49279]: osdmap e107: 8 total, 7 up, 8 in 2024-09-20T18:41:34.521 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:34 smithi174 bash[52692]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4 2024-09-20T18:41:34.521 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:34 smithi174 podman[52707]: 2024-09-20 18:41:34.345608425 +0000 UTC m=+0.546569677 container remove e0ddd37493babbb36273a9d8972ce8dfa0363618f6efbb86ac18bc1ed16df47f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, version=8, vcs-type=git, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-20T18:41:34.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:34 smithi012 ceph-mon[65348]: pgmap v168: 97 pgs: 97 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:41:34.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:34 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:41:34.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:34 smithi012 ceph-mon[65348]: osdmap e107: 8 total, 7 up, 8 in 2024-09-20T18:41:35.007 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:34 smithi174 podman[52798]: 2024-09-20 18:41:34.738866491 +0000 UTC m=+0.111403250 container create 6abbbf3e99bae2beb4c468d76befcfcb3f09b43f62bb728eb06bfbb2653ec3d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:41:35.008 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:34 smithi174 podman[52798]: 2024-09-20 18:41:34.643378305 +0000 UTC m=+0.015915074 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:41:35.008 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:34 smithi174 podman[52798]: 2024-09-20 18:41:34.992228972 +0000 UTC m=+0.364765727 container init 6abbbf3e99bae2beb4c468d76befcfcb3f09b43f62bb728eb06bfbb2653ec3d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-20T18:41:35.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:34 smithi174 podman[52798]: 2024-09-20 18:41:34.996018345 +0000 UTC m=+0.368555110 container start 6abbbf3e99bae2beb4c468d76befcfcb3f09b43f62bb728eb06bfbb2653ec3d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-20T18:41:35.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:35 smithi174 podman[52798]: 2024-09-20 18:41:35.066424568 +0000 UTC m=+0.438961333 container attach 6abbbf3e99bae2beb4c468d76befcfcb3f09b43f62bb728eb06bfbb2653ec3d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:41:35.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:35 smithi174 podman[52798]: 2024-09-20 18:41:35.182071686 +0000 UTC m=+0.554608448 container died 6abbbf3e99bae2beb4c468d76befcfcb3f09b43f62bb728eb06bfbb2653ec3d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-20T18:41:35.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:35 smithi012 ceph-mon[65348]: osdmap e108: 8 total, 7 up, 8 in 2024-09-20T18:41:35.752 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:35 smithi174 ceph-mon[49279]: osdmap e108: 8 total, 7 up, 8 in 2024-09-20T18:41:36.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:35 smithi174 podman[52821]: 2024-09-20 18:41:35.73687201 +0000 UTC m=+0.544318629 container remove 6abbbf3e99bae2beb4c468d76befcfcb3f09b43f62bb728eb06bfbb2653ec3d2 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:41:36.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:35 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-20T18:41:36.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:35 smithi174 systemd[1]: Stopped Ceph osd.4 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:41:36.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:35 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.4.service: Consumed 5min 738ms CPU time. 2024-09-20T18:41:36.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:35 smithi174 systemd[1]: Starting Ceph osd.4 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:41:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:36 smithi174 ceph-mon[49279]: pgmap v171: 97 pgs: 13 active+undersized, 9 stale+active+clean, 5 active+undersized+degraded, 70 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 16/444 objects degraded (3.604%) 2024-09-20T18:41:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:36 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 16/444 objects degraded (3.604%), 5 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:36.521 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 podman[52922]: 2024-09-20 18:41:36.082536842 +0000 UTC m=+0.015865792 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:41:36.521 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 podman[52922]: 2024-09-20 18:41:36.262810998 +0000 UTC m=+0.196139935 container create aa56bfeccaa13d8505004808ac51ee7f45835bc4009430ac1691afcd8eff8d24 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-20T18:41:36.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:36 smithi012 ceph-mon[65348]: pgmap v171: 97 pgs: 13 active+undersized, 9 stale+active+clean, 5 active+undersized+degraded, 70 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 16/444 objects degraded (3.604%) 2024-09-20T18:41:36.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:36 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 16/444 objects degraded (3.604%), 5 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:37.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 podman[52922]: 2024-09-20 18:41:36.532135968 +0000 UTC m=+0.465464913 container init aa56bfeccaa13d8505004808ac51ee7f45835bc4009430ac1691afcd8eff8d24 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-20T18:41:37.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 podman[52922]: 2024-09-20 18:41:36.535753715 +0000 UTC m=+0.469082653 container start aa56bfeccaa13d8505004808ac51ee7f45835bc4009430ac1691afcd8eff8d24 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:41:37.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 podman[52922]: 2024-09-20 18:41:36.607103881 +0000 UTC m=+0.540432828 container attach aa56bfeccaa13d8505004808ac51ee7f45835bc4009430ac1691afcd8eff8d24 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:41:37.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:37.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 bash[52922]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:37.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:37.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:36 smithi174 bash[52922]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:38.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:38 smithi012 ceph-mon[65348]: pgmap v172: 97 pgs: 18 active+undersized, 7 stale+active+clean, 8 active+undersized+degraded, 64 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 31/444 objects degraded (6.982%) 2024-09-20T18:41:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:38 smithi174 ceph-mon[49279]: pgmap v172: 97 pgs: 18 active+undersized, 7 stale+active+clean, 8 active+undersized+degraded, 64 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 31/444 objects degraded (6.982%) 2024-09-20T18:41:39.955 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:41:39.956 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:39.956 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:41:39.956 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:39.956 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:39.956 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:41:39.956 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T18:41:39.956 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T18:41:39.957 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-20T18:41:39.957 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2024-09-20T18:41:39.957 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-20T18:41:39.957 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-20T18:41:40.242 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-20T18:41:40.242 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2024-09-20T18:41:40.242 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T18:41:40.242 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-20T18:41:40.242 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T18:41:40.242 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-20T18:41:40.243 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate[52937]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-20T18:41:40.243 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 bash[52922]: --> ceph-volume lvm activate successful for osd ID: 4 2024-09-20T18:41:40.243 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:39 smithi174 podman[52922]: 2024-09-20 18:41:39.969183722 +0000 UTC m=+3.902512670 container died aa56bfeccaa13d8505004808ac51ee7f45835bc4009430ac1691afcd8eff8d24 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:41:40.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:40 smithi174 ceph-mon[49279]: pgmap v173: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:40.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:40.895 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:40.895 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:40 smithi174 podman[53144]: 2024-09-20 18:41:40.560207323 +0000 UTC m=+0.585107584 container remove aa56bfeccaa13d8505004808ac51ee7f45835bc4009430ac1691afcd8eff8d24 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-activate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-20T18:41:40.895 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:40 smithi174 podman[53202]: 2024-09-20 18:41:40.670339569 +0000 UTC m=+0.017226839 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:41:40.896 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:40 smithi174 podman[53202]: 2024-09-20 18:41:40.777941152 +0000 UTC m=+0.124828412 container create 532d357d165fd76d2a67faa3826c9b87f26fdbaa589b94506102e7d73ea99230 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-20T18:41:40.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:40 smithi012 ceph-mon[65348]: pgmap v173: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:40.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:40.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:41.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:41 smithi174 podman[53202]: 2024-09-20 18:41:41.021817778 +0000 UTC m=+0.368705040 container init 532d357d165fd76d2a67faa3826c9b87f26fdbaa589b94506102e7d73ea99230 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:41:41.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:41 smithi174 podman[53202]: 2024-09-20 18:41:41.02493921 +0000 UTC m=+0.371826471 container start 532d357d165fd76d2a67faa3826c9b87f26fdbaa589b94506102e7d73ea99230 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:41:41.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:41 smithi174 ceph-osd[53237]: -- 172.21.15.174:0/405227214 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x565208cd21e0 con 0x565208c5e000 2024-09-20T18:41:41.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:41 smithi174 bash[53202]: 532d357d165fd76d2a67faa3826c9b87f26fdbaa589b94506102e7d73ea99230 2024-09-20T18:41:41.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:41 smithi174 systemd[1]: Started Ceph osd.4 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:41:42.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:42 smithi012 ceph-mon[65348]: pgmap v174: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 50/444 objects degraded (11.261%) 2024-09-20T18:41:42.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:42.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:42 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 50/444 objects degraded (11.261%), 11 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:42 smithi174 ceph-mon[49279]: pgmap v174: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 50/444 objects degraded (11.261%) 2024-09-20T18:41:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:42 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 50/444 objects degraded (11.261%), 11 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:43.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:42 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:41:42.911+0000 7f0ffd380640 -1 Falling back to public interface 2024-09-20T18:41:44.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:44 smithi174 ceph-mon[49279]: pgmap v175: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 211 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:44.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:44 smithi012 ceph-mon[65348]: pgmap v175: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 211 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:45.638 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:45.638 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:45.638 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:45.638 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:45 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:45.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:45.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:45.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:45.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:45 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:46.698 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:46 smithi174 ceph-mon[49279]: pgmap v176: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 191 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:46.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:46 smithi012 ceph-mon[65348]: pgmap v176: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 191 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-mon[49279]: pgmap v177: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:41:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:41:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:48.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:48.684 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:48 smithi012 ceph-mon[65348]: pgmap v177: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:48.685 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:48.685 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:48.685 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:41:48.685 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:41:48.685 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:48.685 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:48 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:49.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:48 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:41:48.950+0000 7f0ffd380640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-20T18:41:49.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:49.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:49.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:49.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:49.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T18:41:49.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:49 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T18:41:49.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:49 smithi012 ceph-mon[65348]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-20T18:41:49.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:41:49.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:49.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:49.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:49.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T18:41:49.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:49 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T18:41:49.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:49 smithi174 ceph-mon[49279]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-20T18:41:50.436 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:41:50.124+0000 7f0ffd380640 -1 osd.4 106 log_to_monitors true 2024-09-20T18:41:50.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:50 smithi012 ceph-mon[65348]: pgmap v178: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:50.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:50 smithi012 ceph-mon[65348]: from='osd.4 [v2:172.21.15.174:6800/3809198964,v1:172.21.15.174:6801/3809198964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T18:41:50.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:50 smithi012 ceph-mon[65348]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T18:41:50.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:50 smithi174 ceph-mon[49279]: pgmap v178: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:50.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:50 smithi174 ceph-mon[49279]: from='osd.4 [v2:172.21.15.174:6800/3809198964,v1:172.21.15.174:6801/3809198964]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T18:41:50.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:50 smithi174 ceph-mon[49279]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-20T18:41:51.271 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:41:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:41:50.878+0000 7f0ff1122640 -1 osd.4 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:41:51.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:51 smithi174 ceph-mon[49279]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T18:41:51.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:51 smithi174 ceph-mon[49279]: osdmap e109: 8 total, 7 up, 8 in 2024-09-20T18:41:51.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:51 smithi174 ceph-mon[49279]: from='osd.4 [v2:172.21.15.174:6800/3809198964,v1:172.21.15.174:6801/3809198964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:41:51.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:51 smithi174 ceph-mon[49279]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:41:51.717 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:51 smithi174 ceph-mon[49279]: from='osd.4 ' entity='osd.4' 2024-09-20T18:41:51.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:51 smithi012 ceph-mon[65348]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-20T18:41:51.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:51 smithi012 ceph-mon[65348]: osdmap e109: 8 total, 7 up, 8 in 2024-09-20T18:41:51.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:51 smithi012 ceph-mon[65348]: from='osd.4 [v2:172.21.15.174:6800/3809198964,v1:172.21.15.174:6801/3809198964]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:41:51.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:51 smithi012 ceph-mon[65348]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:41:51.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:51 smithi012 ceph-mon[65348]: from='osd.4 ' entity='osd.4' 2024-09-20T18:41:52.279 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:41:52.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:52 smithi012 ceph-mon[65348]: pgmap v180: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:52.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:52 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:41:52.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:52 smithi012 ceph-mon[65348]: osd.4 [v2:172.21.15.174:6800/3809198964,v1:172.21.15.174:6801/3809198964] boot 2024-09-20T18:41:52.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:52 smithi012 ceph-mon[65348]: osdmap e110: 8 total, 8 up, 8 in 2024-09-20T18:41:52.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T18:41:52.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:52 smithi174 ceph-mon[49279]: pgmap v180: 97 pgs: 30 active+undersized, 11 active+undersized+degraded, 56 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 50/444 objects degraded (11.261%) 2024-09-20T18:41:52.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:52 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:41:52.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:52 smithi174 ceph-mon[49279]: osd.4 [v2:172.21.15.174:6800/3809198964,v1:172.21.15.174:6801/3809198964] boot 2024-09-20T18:41:52.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:52 smithi174 ceph-mon[49279]: osdmap e110: 8 total, 8 up, 8 in 2024-09-20T18:41:52.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-20T18:41:53.011 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:41:53.011 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (6m) 44s ago 21m 16.1M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:41:53.011 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (3m) 44s ago 21m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:41:53.011 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (3m) 8s ago 20m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:41:53.011 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (6m) 44s ago 21m 45.4M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:41:53.011 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (15m) 44s ago 15m 41.1M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:41:53.012 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (15m) 8s ago 15m 16.6M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:41:53.012 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (6m) 44s ago 22m 522M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:41:53.012 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (5m) 8s ago 20m 457M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:41:53.012 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (4m) 44s ago 22m 53.1M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:41:53.012 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (4m) 8s ago 20m 43.1M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:41:53.012 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (14m) 44s ago 14m 265M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:41:53.013 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (14m) 8s ago 14m 86.4M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:41:53.013 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (6m) 44s ago 21m 9.92M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:41:53.013 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 8s ago 20m 11.5M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:41:53.013 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (2m) 44s ago 19m 252M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:41:53.013 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (2m) 44s ago 19m 209M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:41:53.013 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (90s) 44s ago 18m 193M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:41:53.013 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (50s) 44s ago 18m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:41:53.014 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (12s) 8s ago 17m 11.7M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:41:53.014 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (17m) 8s ago 17m 630M 2704M 17.2.0 e1d6a67b021e 69d3c04f15bc 2024-09-20T18:41:53.014 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (17m) 8s ago 17m 555M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:41:53.014 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (16m) 8s ago 16m 550M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:41:53.014 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (5m) 44s ago 20m 55.3M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:41:53.449 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:41:53.449 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:41:53.449 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:41:53.449 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:53.449 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:41:53.449 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 5 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 9 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:41:53.450 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:41:53.817 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:41:53.817 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:41:53.817 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "11/23 daemons upgraded", 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:41:53.818 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:41:54.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:53 smithi012 ceph-mon[65348]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:54.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:53 smithi012 ceph-mon[65348]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:54.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:53 smithi012 ceph-mon[65348]: osdmap e111: 8 total, 8 up, 8 in 2024-09-20T18:41:54.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:53 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2193018266' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN Degraded data redundancy: 37/444 objects degraded (8.333%), 9 pgs degraded 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 37/444 objects degraded (8.333%), 9 pgs degraded 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.5 is active+undersized+degraded, acting [2,0] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-20T18:41:54.257 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-20T18:41:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:53 smithi174 ceph-mon[49279]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:53 smithi174 ceph-mon[49279]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:53 smithi174 ceph-mon[49279]: osdmap e111: 8 total, 8 up, 8 in 2024-09-20T18:41:54.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:53 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2193018266' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:41:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:54 smithi012 ceph-mon[65348]: pgmap v183: 97 pgs: 6 peering, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 37/444 objects degraded (8.333%) 2024-09-20T18:41:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:54 smithi012 ceph-mon[65348]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:54 smithi012 ceph-mon[65348]: from='client.34354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:54 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 37/444 objects degraded (8.333%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:54 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/270896233' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:41:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:54 smithi174 ceph-mon[49279]: pgmap v183: 97 pgs: 6 peering, 26 active+undersized, 9 active+undersized+degraded, 56 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 37/444 objects degraded (8.333%) 2024-09-20T18:41:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:54 smithi174 ceph-mon[49279]: from='client.34346 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:54 smithi174 ceph-mon[49279]: from='client.34354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:41:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:54 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 37/444 objects degraded (8.333%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T18:41:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:54 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/270896233' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:41:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:41:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:41:57.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:56 smithi012 ceph-mon[65348]: pgmap v184: 97 pgs: 6 peering, 17 active+undersized, 3 active+undersized+degraded, 71 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 15/444 objects degraded (3.378%) 2024-09-20T18:41:57.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:56 smithi174 ceph-mon[49279]: pgmap v184: 97 pgs: 6 peering, 17 active+undersized, 3 active+undersized+degraded, 71 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 15/444 objects degraded (3.378%) 2024-09-20T18:41:58.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:57 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/444 objects degraded (3.378%), 3 pgs degraded) 2024-09-20T18:41:58.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:57 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:41:58.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:57 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/444 objects degraded (3.378%), 3 pgs degraded) 2024-09-20T18:41:58.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:57 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:41:59.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:58 smithi012 ceph-mon[65348]: pgmap v185: 97 pgs: 6 peering, 91 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-09-20T18:41:59.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:58 smithi174 ceph-mon[49279]: pgmap v185: 97 pgs: 6 peering, 91 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 313 B/s rd, 0 op/s 2024-09-20T18:42:00.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:41:59 smithi012 ceph-mon[65348]: pgmap v186: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:42:00.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:41:59 smithi174 ceph-mon[49279]: pgmap v186: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:42:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:02 smithi174 ceph-mon[49279]: pgmap v187: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 225 B/s rd, 0 op/s 2024-09-20T18:42:02.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:02 smithi012 ceph-mon[65348]: pgmap v187: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 225 B/s rd, 0 op/s 2024-09-20T18:42:04.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:04 smithi174 ceph-mon[49279]: pgmap v188: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-09-20T18:42:04.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:04 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T18:42:04.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:04 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T18:42:04.306 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:04 smithi174 ceph-mon[49279]: Upgrade: osd.5 is safe to restart 2024-09-20T18:42:04.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:04 smithi012 ceph-mon[65348]: pgmap v188: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 203 B/s rd, 0 op/s 2024-09-20T18:42:04.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:04 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-20T18:42:04.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:04 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-20T18:42:04.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:04 smithi012 ceph-mon[65348]: Upgrade: osd.5 is safe to restart 2024-09-20T18:42:05.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:05.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:05.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T18:42:05.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:42:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-20T18:42:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:42:06.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:06 smithi174 ceph-mon[49279]: pgmap v189: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:42:06.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:06 smithi174 ceph-mon[49279]: Upgrade: Updating osd.5 2024-09-20T18:42:06.930 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:06 smithi174 ceph-mon[49279]: Deploying daemon osd.5 on smithi174 2024-09-20T18:42:06.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:06 smithi012 ceph-mon[65348]: pgmap v189: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:42:06.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:06 smithi012 ceph-mon[65348]: Upgrade: Updating osd.5 2024-09-20T18:42:06.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:06 smithi012 ceph-mon[65348]: Deploying daemon osd.5 on smithi174 2024-09-20T18:42:08.155 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:07 smithi174 systemd[1]: Stopping Ceph osd.5 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:42:08.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:08 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[32860]: 2024-09-20T18:42:08.187+0000 7fe1e82df700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:42:08.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:08 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[32860]: 2024-09-20T18:42:08.187+0000 7fe1e82df700 -1 osd.5 111 *** Got signal Terminated *** 2024-09-20T18:42:08.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:08 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[32860]: 2024-09-20T18:42:08.187+0000 7fe1e82df700 -1 osd.5 111 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:42:08.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:08 smithi012 ceph-mon[65348]: pgmap v190: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:42:08.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:08 smithi012 ceph-mon[65348]: osd.5 marked itself down and dead 2024-09-20T18:42:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:08 smithi174 ceph-mon[49279]: pgmap v190: 97 pgs: 97 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:42:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:08 smithi174 ceph-mon[49279]: osd.5 marked itself down and dead 2024-09-20T18:42:09.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:09 smithi174 podman[54806]: 2024-09-20 18:42:09.235881618 +0000 UTC m=+1.171106354 container died 69d3c04f15bcf1f87f7a3fd87458bf9ad637e47b4031be7a4326260a5f0986b0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, version=8, vcs-type=git, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-20T18:42:09.888 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:09 smithi174 podman[54806]: 2024-09-20 18:42:09.596376936 +0000 UTC m=+1.531601671 container cleanup 69d3c04f15bcf1f87f7a3fd87458bf9ad637e47b4031be7a4326260a5f0986b0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, io.openshift.tags=base centos centos-stream, ceph=True, maintainer=Guillaume Abrioux , io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, vendor=Red Hat, Inc., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-20T18:42:09.888 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:09 smithi174 bash[54806]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5 2024-09-20T18:42:09.888 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: osdmap e112: 8 total, 7 up, 8 in 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 2]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 5]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]} : dispatch 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:09.889 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 2]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 5]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]}]': finished 2024-09-20T18:42:09.890 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:09 smithi174 ceph-mon[49279]: osdmap e113: 8 total, 7 up, 8 in 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: osdmap e112: 8 total, 7 up, 8 in 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 2]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 5]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]} : dispatch 2024-09-20T18:42:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]} : dispatch 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]} : dispatch 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2", "id": [3, 2]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [4, 2]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.11", "id": [4, 2]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.14", "id": [4, 5]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.15", "id": [7, 5]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1a", "id": [4, 5]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1b", "id": [4, 5]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]': finished 2024-09-20T18:42:10.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 0]}]': finished 2024-09-20T18:42:10.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 2]}]': finished 2024-09-20T18:42:10.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:09 smithi012 ceph-mon[65348]: osdmap e113: 8 total, 7 up, 8 in 2024-09-20T18:42:10.240 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:09 smithi174 podman[54833]: 2024-09-20 18:42:09.8728748 +0000 UTC m=+0.633484145 container remove 69d3c04f15bcf1f87f7a3fd87458bf9ad637e47b4031be7a4326260a5f0986b0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, version=8, release=754, ceph=True, GIT_CLEAN=True, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870) 2024-09-20T18:42:10.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:10 smithi174 podman[54921]: 2024-09-20 18:42:10.128099004 +0000 UTC m=+0.015652069 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:42:10.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:10 smithi174 podman[54921]: 2024-09-20 18:42:10.256990193 +0000 UTC m=+0.144543248 container create 0fd1683705f35e456dc56856898ee0c5996f94cf908d0b5c075cc49956bc2c21 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:42:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:10 smithi174 ceph-mon[49279]: pgmap v192: 97 pgs: 12 stale+active+clean, 85 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:42:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:10 smithi174 ceph-mon[49279]: osdmap e114: 8 total, 7 up, 8 in 2024-09-20T18:42:11.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:10 smithi174 podman[54921]: 2024-09-20 18:42:10.537381103 +0000 UTC m=+0.424934163 container init 0fd1683705f35e456dc56856898ee0c5996f94cf908d0b5c075cc49956bc2c21 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:42:11.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:10 smithi174 podman[54921]: 2024-09-20 18:42:10.540890913 +0000 UTC m=+0.428443970 container start 0fd1683705f35e456dc56856898ee0c5996f94cf908d0b5c075cc49956bc2c21 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-20T18:42:11.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:10 smithi174 podman[54921]: 2024-09-20 18:42:10.617687543 +0000 UTC m=+0.505240609 container attach 0fd1683705f35e456dc56856898ee0c5996f94cf908d0b5c075cc49956bc2c21 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-20T18:42:11.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:10 smithi174 podman[54921]: 2024-09-20 18:42:10.728031764 +0000 UTC m=+0.615584827 container died 0fd1683705f35e456dc56856898ee0c5996f94cf908d0b5c075cc49956bc2c21 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD) 2024-09-20T18:42:11.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:10 smithi012 ceph-mon[65348]: pgmap v192: 97 pgs: 12 stale+active+clean, 85 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:42:11.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:10 smithi012 ceph-mon[65348]: osdmap e114: 8 total, 7 up, 8 in 2024-09-20T18:42:11.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:11 smithi174 podman[54944]: 2024-09-20 18:42:11.28808961 +0000 UTC m=+0.553301905 container remove 0fd1683705f35e456dc56856898ee0c5996f94cf908d0b5c075cc49956bc2c21 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-20T18:42:11.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:11 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-20T18:42:11.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:11 smithi174 systemd[1]: Stopped Ceph osd.5 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:42:11.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:11 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.5.service: Consumed 5min 25.638s CPU time. 2024-09-20T18:42:11.872 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:11 smithi174 systemd[1]: Starting Ceph osd.5 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:42:11.872 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:11 smithi174 podman[55043]: 2024-09-20 18:42:11.619167305 +0000 UTC m=+0.016711476 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:42:11.872 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:11 smithi174 podman[55043]: 2024-09-20 18:42:11.722522903 +0000 UTC m=+0.120067073 container create 77bf5c47a6831d2fff19f25e4b914ac4292668236c9051e8a47b561452101022 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-20T18:42:12.123 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:12 smithi174 podman[55043]: 2024-09-20 18:42:12.008535079 +0000 UTC m=+0.406079249 container init 77bf5c47a6831d2fff19f25e4b914ac4292668236c9051e8a47b561452101022 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-20T18:42:12.123 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:12 smithi174 podman[55043]: 2024-09-20 18:42:12.011918936 +0000 UTC m=+0.409463103 container start 77bf5c47a6831d2fff19f25e4b914ac4292668236c9051e8a47b561452101022 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:42:12.124 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:12 smithi174 podman[55043]: 2024-09-20 18:42:12.108582343 +0000 UTC m=+0.506126526 container attach 77bf5c47a6831d2fff19f25e4b914ac4292668236c9051e8a47b561452101022 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0) 2024-09-20T18:42:12.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:12 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:12.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:12 smithi174 bash[55043]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:12.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:12 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:12.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:12 smithi174 bash[55043]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:12.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:12 smithi012 ceph-mon[65348]: pgmap v195: 97 pgs: 6 peering, 11 active+undersized, 5 stale+active+clean, 7 active+undersized+degraded, 68 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 26/444 objects degraded (5.856%) 2024-09-20T18:42:12.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:12 smithi012 ceph-mon[65348]: osdmap e115: 8 total, 7 up, 8 in 2024-09-20T18:42:12.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:12 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 26/444 objects degraded (5.856%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:12 smithi174 ceph-mon[49279]: pgmap v195: 97 pgs: 6 peering, 11 active+undersized, 5 stale+active+clean, 7 active+undersized+degraded, 68 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 26/444 objects degraded (5.856%) 2024-09-20T18:42:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:12 smithi174 ceph-mon[49279]: osdmap e115: 8 total, 7 up, 8 in 2024-09-20T18:42:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:12 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 26/444 objects degraded (5.856%), 7 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:15.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:14 smithi174 ceph-mon[49279]: pgmap v197: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 6 peering, 17 active+undersized, 1 stale+active+clean, 11 active+undersized+degraded, 58 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 480 B/s rd, 0 op/s; 715/444 objects degraded (161.036%); 672/444 objects misplaced (151.351%) 2024-09-20T18:42:15.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:14 smithi174 ceph-mon[49279]: osdmap e116: 8 total, 7 up, 8 in 2024-09-20T18:42:15.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:14 smithi012 ceph-mon[65348]: pgmap v197: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 6 peering, 17 active+undersized, 1 stale+active+clean, 11 active+undersized+degraded, 58 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 480 B/s rd, 0 op/s; 715/444 objects degraded (161.036%); 672/444 objects misplaced (151.351%) 2024-09-20T18:42:15.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:14 smithi012 ceph-mon[65348]: osdmap e116: 8 total, 7 up, 8 in 2024-09-20T18:42:15.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:42:15.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:15.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:42:15.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:15.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:15.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:15.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T18:42:15.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T18:42:15.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-20T18:42:15.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2024-09-20T18:42:15.826 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-mon[49279]: osdmap e117: 8 total, 7 up, 8 in 2024-09-20T18:42:15.826 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-20T18:42:15.826 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-20T18:42:15.826 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-20T18:42:15.826 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2024-09-20T18:42:15.827 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T18:42:15.827 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-20T18:42:15.827 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T18:42:15.827 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-20T18:42:15.827 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate[55058]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-20T18:42:15.827 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 bash[55043]: --> ceph-volume lvm activate successful for osd ID: 5 2024-09-20T18:42:15.827 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:15 smithi174 podman[55043]: 2024-09-20 18:42:15.538940504 +0000 UTC m=+3.936484669 container died 77bf5c47a6831d2fff19f25e4b914ac4292668236c9051e8a47b561452101022 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:42:16.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:15 smithi012 ceph-mon[65348]: osdmap e117: 8 total, 7 up, 8 in 2024-09-20T18:42:16.519 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 podman[55273]: 2024-09-20 18:42:16.187695632 +0000 UTC m=+0.638430118 container remove 77bf5c47a6831d2fff19f25e4b914ac4292668236c9051e8a47b561452101022 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-activate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-20T18:42:16.520 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 podman[55334]: 2024-09-20 18:42:16.388368089 +0000 UTC m=+0.109350110 container create b38346191a9dd318608fd671eeaa0830c2899de4848faf91b589f2344fe51566 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-20T18:42:16.520 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 podman[55334]: 2024-09-20 18:42:16.294466274 +0000 UTC m=+0.015448292 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:42:17.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:16 smithi174 ceph-mon[49279]: pgmap v200: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 3 peering, 17 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 480 B/s rd, 0 op/s; 720/444 objects degraded (162.162%); 673/444 objects misplaced (151.577%); 0 B/s, 166 objects/s recovering 2024-09-20T18:42:17.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:16 smithi174 ceph-mon[49279]: osdmap e118: 8 total, 7 up, 8 in 2024-09-20T18:42:17.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 podman[55334]: 2024-09-20 18:42:16.615423185 +0000 UTC m=+0.336405203 container init b38346191a9dd318608fd671eeaa0830c2899de4848faf91b589f2344fe51566 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-20T18:42:17.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 podman[55334]: 2024-09-20 18:42:16.618818822 +0000 UTC m=+0.339800840 container start b38346191a9dd318608fd671eeaa0830c2899de4848faf91b589f2344fe51566 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-20T18:42:17.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 ceph-osd[55354]: -- 172.21.15.174:0/2188833986 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c4b70b01e0 con 0x55c4b624fc00 2024-09-20T18:42:17.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 bash[55334]: b38346191a9dd318608fd671eeaa0830c2899de4848faf91b589f2344fe51566 2024-09-20T18:42:17.022 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:16 smithi174 systemd[1]: Started Ceph osd.5 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:42:17.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:16 smithi012 ceph-mon[65348]: pgmap v200: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 3 peering, 17 active+undersized, 13 active+undersized+degraded, 58 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 480 B/s rd, 0 op/s; 720/444 objects degraded (162.162%); 673/444 objects misplaced (151.577%); 0 B/s, 166 objects/s recovering 2024-09-20T18:42:17.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:16 smithi012 ceph-mon[65348]: osdmap e118: 8 total, 7 up, 8 in 2024-09-20T18:42:18.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:17 smithi174 ceph-mon[49279]: osdmap e119: 8 total, 7 up, 8 in 2024-09-20T18:42:18.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:18.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:18.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:42:18.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:17 smithi012 ceph-mon[65348]: osdmap e119: 8 total, 7 up, 8 in 2024-09-20T18:42:18.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:18.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:18.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:42:18.304 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:18 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:42:18.033+0000 7f53d9046640 -1 Falling back to public interface 2024-09-20T18:42:18.873 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:18 smithi174 ceph-mon[49279]: pgmap v203: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 17 active+undersized, 13 active+undersized+degraded, 60 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 720/444 objects degraded (162.162%); 673/444 objects misplaced (151.577%); 0 B/s, 177 objects/s recovering 2024-09-20T18:42:19.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:18 smithi012 ceph-mon[65348]: pgmap v203: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 5 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 17 active+undersized, 13 active+undersized+degraded, 60 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 720/444 objects degraded (162.162%); 673/444 objects misplaced (151.577%); 0 B/s, 177 objects/s recovering 2024-09-20T18:42:20.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:19 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 44/444 objects degraded (9.910%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:20.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:19 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 44/444 objects degraded (9.910%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:21.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:20 smithi174 ceph-mon[49279]: pgmap v204: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 391 objects/s recovering 2024-09-20T18:42:21.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:21.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:21.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:21.172 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:21.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:20 smithi012 ceph-mon[65348]: pgmap v204: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 391 objects/s recovering 2024-09-20T18:42:21.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:21.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:21.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:21.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:23.207 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:22 smithi174 ceph-mon[49279]: pgmap v205: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 329 objects/s recovering 2024-09-20T18:42:23.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:22 smithi012 ceph-mon[65348]: pgmap v205: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 329 objects/s recovering 2024-09-20T18:42:24.637 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:42:25.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: pgmap v206: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 168 objects/s recovering 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T18:42:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: pgmap v206: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 168 objects/s recovering 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:42:25.227 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:25.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:25.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:25.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T18:42:25.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:25.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:25.358 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:42:25.358 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (7m) 77s ago 21m 16.1M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:42:25.358 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (4m) 77s ago 21m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:42:25.358 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (4m) 5s ago 21m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:42:25.358 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (7m) 77s ago 21m 45.4M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (15m) 77s ago 15m 41.1M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (15m) 5s ago 15m 16.7M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (6m) 77s ago 23m 522M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (5m) 5s ago 21m 457M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (5m) 77s ago 23m 53.1M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (4m) 5s ago 21m 44.7M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (15m) 77s ago 15m 265M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (15m) 5s ago 15m 86.5M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (7m) 77s ago 21m 9.92M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (6m) 5s ago 21m 10.9M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (3m) 77s ago 20m 252M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (2m) 77s ago 19m 209M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (2m) 77s ago 19m 193M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:42:25.359 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (83s) 77s ago 18m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:42:25.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (44s) 5s ago 18m 232M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:42:25.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (9s) 5s ago 18m 11.8M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b38346191a9d 2024-09-20T18:42:25.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (17m) 5s ago 17m 558M 2704M 17.2.0 e1d6a67b021e b4c31683bf4d 2024-09-20T18:42:25.360 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (17m) 5s ago 17m 580M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:42:25.360 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (5m) 77s ago 20m 55.3M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:42:25.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:42:25.209+0000 7f53d9046640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 5 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:25.796 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:42:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:42:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:42:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-20T18:42:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 9 2024-09-20T18:42:25.797 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:42:25.797 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "12/23 daemons upgraded", 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:42:26.167 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:42:26.168 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:42:26.168 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:25 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T18:42:26.168 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:25 smithi012 ceph-mon[65348]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T18:42:26.168 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:25 smithi012 ceph-mon[65348]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:26.168 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:25 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/34913450' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:26.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:25 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T18:42:26.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:25 smithi174 ceph-mon[49279]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T18:42:26.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:25 smithi174 ceph-mon[49279]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:26.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:25 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/34913450' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 44/444 objects degraded (9.910%), 13 pgs degraded 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: osd.5 (root=default,host=smithi174) is down 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 44/444 objects degraded (9.910%), 13 pgs degraded 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-20T18:42:26.604 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-20T18:42:26.605 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-20T18:42:26.605 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-20T18:42:26.605 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-20T18:42:26.605 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-20T18:42:26.771 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:42:26.296+0000 7f53d9046640 -1 osd.5 111 log_to_monitors true 2024-09-20T18:42:27.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:26 smithi012 ceph-mon[65348]: pgmap v207: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 221 B/s rd, 0 op/s; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 145 objects/s recovering 2024-09-20T18:42:27.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:26 smithi012 ceph-mon[65348]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:27.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:26 smithi012 ceph-mon[65348]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:27.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:26 smithi012 ceph-mon[65348]: from='osd.5 [v2:172.21.15.174:6808/2053322680,v1:172.21.15.174:6809/2053322680]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T18:42:27.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:26 smithi012 ceph-mon[65348]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T18:42:27.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:26 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2316241773' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:42:27.271 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:42:26.913+0000 7f53d0df0640 -1 osd.5 111 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:42:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-mon[49279]: pgmap v207: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 221 B/s rd, 0 op/s; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 145 objects/s recovering 2024-09-20T18:42:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-mon[49279]: from='client.34366 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-mon[49279]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-mon[49279]: from='osd.5 [v2:172.21.15.174:6808/2053322680,v1:172.21.15.174:6809/2053322680]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T18:42:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-mon[49279]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-20T18:42:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:26 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2316241773' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:42:28.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:27 smithi012 ceph-mon[65348]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:28.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:27 smithi012 ceph-mon[65348]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T18:42:28.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:27 smithi012 ceph-mon[65348]: from='osd.5 [v2:172.21.15.174:6808/2053322680,v1:172.21.15.174:6809/2053322680]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:42:28.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:27 smithi012 ceph-mon[65348]: osdmap e120: 8 total, 7 up, 8 in 2024-09-20T18:42:28.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:27 smithi012 ceph-mon[65348]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:42:28.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:27 smithi174 ceph-mon[49279]: from='client.34378 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:28.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:27 smithi174 ceph-mon[49279]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-20T18:42:28.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:27 smithi174 ceph-mon[49279]: from='osd.5 [v2:172.21.15.174:6808/2053322680,v1:172.21.15.174:6809/2053322680]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:42:28.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:27 smithi174 ceph-mon[49279]: osdmap e120: 8 total, 7 up, 8 in 2024-09-20T18:42:28.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:27 smithi174 ceph-mon[49279]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:42:29.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:28 smithi012 ceph-mon[65348]: pgmap v209: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 134 objects/s recovering 2024-09-20T18:42:29.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:28 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:42:29.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:28 smithi012 ceph-mon[65348]: osd.5 [v2:172.21.15.174:6808/2053322680,v1:172.21.15.174:6809/2053322680] boot 2024-09-20T18:42:29.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:28 smithi012 ceph-mon[65348]: osdmap e121: 8 total, 8 up, 8 in 2024-09-20T18:42:29.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:28 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T18:42:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:28 smithi174 ceph-mon[49279]: pgmap v209: 97 pgs: 6 active+clean+remapped, 17 active+undersized, 13 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 44/444 objects degraded (9.910%); 1/444 objects misplaced (0.225%); 0 B/s, 134 objects/s recovering 2024-09-20T18:42:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:28 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:42:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:28 smithi174 ceph-mon[49279]: osd.5 [v2:172.21.15.174:6808/2053322680,v1:172.21.15.174:6809/2053322680] boot 2024-09-20T18:42:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:28 smithi174 ceph-mon[49279]: osdmap e121: 8 total, 8 up, 8 in 2024-09-20T18:42:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:28 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-20T18:42:30.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:29 smithi012 ceph-mon[65348]: osdmap e122: 8 total, 8 up, 8 in 2024-09-20T18:42:30.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:29 smithi174 ceph-mon[49279]: osdmap e122: 8 total, 8 up, 8 in 2024-09-20T18:42:31.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:30 smithi012 ceph-mon[65348]: pgmap v212: 97 pgs: 1 remapped+peering, 1 peering, 5 active+clean+remapped, 17 active+undersized, 12 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43/444 objects degraded (9.685%); 1/444 objects misplaced (0.225%) 2024-09-20T18:42:31.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:30 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 43/444 objects degraded (9.685%), 12 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:31.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:30 smithi012 ceph-mon[65348]: osdmap e123: 8 total, 8 up, 8 in 2024-09-20T18:42:31.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:30 smithi174 ceph-mon[49279]: pgmap v212: 97 pgs: 1 remapped+peering, 1 peering, 5 active+clean+remapped, 17 active+undersized, 12 active+undersized+degraded, 61 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43/444 objects degraded (9.685%); 1/444 objects misplaced (0.225%) 2024-09-20T18:42:31.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:30 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 43/444 objects degraded (9.685%), 12 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:31.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:30 smithi174 ceph-mon[49279]: osdmap e123: 8 total, 8 up, 8 in 2024-09-20T18:42:32.503 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:32 smithi174 ceph-mon[49279]: pgmap v214: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 peering, 3 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 72 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 706/444 objects degraded (159.009%); 1/444 objects misplaced (0.225%) 2024-09-20T18:42:32.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:32 smithi174 ceph-mon[49279]: osdmap e124: 8 total, 8 up, 8 in 2024-09-20T18:42:32.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:32 smithi012 ceph-mon[65348]: pgmap v214: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 peering, 3 active+clean+remapped, 11 active+undersized, 7 active+undersized+degraded, 72 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 706/444 objects degraded (159.009%); 1/444 objects misplaced (0.225%) 2024-09-20T18:42:32.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:32 smithi012 ceph-mon[65348]: osdmap e124: 8 total, 8 up, 8 in 2024-09-20T18:42:33.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:33 smithi174 ceph-mon[49279]: osdmap e125: 8 total, 8 up, 8 in 2024-09-20T18:42:33.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:33 smithi174 ceph-mon[49279]: osdmap e126: 8 total, 8 up, 8 in 2024-09-20T18:42:33.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:33 smithi012 ceph-mon[65348]: osdmap e125: 8 total, 8 up, 8 in 2024-09-20T18:42:33.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:33 smithi012 ceph-mon[65348]: osdmap e126: 8 total, 8 up, 8 in 2024-09-20T18:42:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:34 smithi174 ceph-mon[49279]: pgmap v217: 97 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 2 peering, 91 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 505 B/s rd, 0 op/s; 680/444 objects degraded (153.153%); 0 B/s, 189 objects/s recovering 2024-09-20T18:42:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:34 smithi174 ceph-mon[49279]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T18:42:34.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:34 smithi174 ceph-mon[49279]: osdmap e127: 8 total, 8 up, 8 in 2024-09-20T18:42:34.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:34 smithi012 ceph-mon[65348]: pgmap v217: 97 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 2 peering, 91 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 505 B/s rd, 0 op/s; 680/444 objects degraded (153.153%); 0 B/s, 189 objects/s recovering 2024-09-20T18:42:34.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:34 smithi012 ceph-mon[65348]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-20T18:42:34.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:34 smithi012 ceph-mon[65348]: osdmap e127: 8 total, 8 up, 8 in 2024-09-20T18:42:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:36 smithi174 ceph-mon[49279]: pgmap v220: 97 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 93 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 680/444 objects degraded (153.153%); 0 B/s, 191 objects/s recovering 2024-09-20T18:42:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:36 smithi174 ceph-mon[49279]: osdmap e128: 8 total, 8 up, 8 in 2024-09-20T18:42:36.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:36 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 680/444 objects degraded (153.153%), 1 pg degraded (PG_DEGRADED) 2024-09-20T18:42:36.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:36 smithi012 ceph-mon[65348]: pgmap v220: 97 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 peering, 93 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 680/444 objects degraded (153.153%); 0 B/s, 191 objects/s recovering 2024-09-20T18:42:36.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:36 smithi012 ceph-mon[65348]: osdmap e128: 8 total, 8 up, 8 in 2024-09-20T18:42:36.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:36 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 680/444 objects degraded (153.153%), 1 pg degraded (PG_DEGRADED) 2024-09-20T18:42:37.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:37 smithi012 ceph-mon[65348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T18:42:37.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:37 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 680/444 objects degraded (153.153%), 1 pg degraded) 2024-09-20T18:42:37.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:37 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:42:37.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:37 smithi174 ceph-mon[49279]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-20T18:42:37.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:37 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 680/444 objects degraded (153.153%), 1 pg degraded) 2024-09-20T18:42:37.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:37 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:42:38.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:38 smithi012 ceph-mon[65348]: pgmap v222: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 432 B/s rd, 0 op/s; 0 B/s, 420 objects/s recovering 2024-09-20T18:42:38.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:38 smithi174 ceph-mon[49279]: pgmap v222: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 432 B/s rd, 0 op/s; 0 B/s, 420 objects/s recovering 2024-09-20T18:42:40.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:40 smithi012 ceph-mon[65348]: pgmap v223: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 204 objects/s recovering 2024-09-20T18:42:40.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T18:42:40.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:40 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T18:42:40.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:40 smithi012 ceph-mon[65348]: Upgrade: osd.6 is safe to restart 2024-09-20T18:42:40.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:40.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:40.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:40 smithi174 ceph-mon[49279]: pgmap v223: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 204 objects/s recovering 2024-09-20T18:42:40.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-20T18:42:40.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:40 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-20T18:42:40.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:40 smithi174 ceph-mon[49279]: Upgrade: osd.6 is safe to restart 2024-09-20T18:42:40.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:40.747 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:42.061 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:41 smithi174 ceph-mon[49279]: Upgrade: Updating osd.6 2024-09-20T18:42:42.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:41 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:42.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:41 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T18:42:42.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:41 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:42:42.062 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:41 smithi174 ceph-mon[49279]: Deploying daemon osd.6 on smithi174 2024-09-20T18:42:42.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:41 smithi012 ceph-mon[65348]: Upgrade: Updating osd.6 2024-09-20T18:42:42.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:41 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:42.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:41 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-20T18:42:42.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:41 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:42:42.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:41 smithi012 ceph-mon[65348]: Deploying daemon osd.6 on smithi174 2024-09-20T18:42:43.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:42 smithi174 ceph-mon[49279]: pgmap v224: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 158 objects/s recovering 2024-09-20T18:42:43.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:42 smithi012 ceph-mon[65348]: pgmap v224: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s, 158 objects/s recovering 2024-09-20T18:42:43.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:43 smithi174 systemd[1]: Stopping Ceph osd.6 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:42:44.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[35875]: 2024-09-20T18:42:43.857+0000 7f65cda84700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:42:44.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[35875]: 2024-09-20T18:42:43.857+0000 7f65cda84700 -1 osd.6 128 *** Got signal Terminated *** 2024-09-20T18:42:44.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[35875]: 2024-09-20T18:42:43.857+0000 7f65cda84700 -1 osd.6 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:42:45.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:44 smithi174 ceph-mon[49279]: pgmap v225: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 0 B/s, 140 objects/s recovering 2024-09-20T18:42:45.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:44 smithi174 ceph-mon[49279]: osd.6 marked itself down and dead 2024-09-20T18:42:45.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:44 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:45.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:44 smithi012 ceph-mon[65348]: pgmap v225: 97 pgs: 97 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 234 B/s rd, 0 op/s; 0 B/s, 140 objects/s recovering 2024-09-20T18:42:45.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:44 smithi012 ceph-mon[65348]: osd.6 marked itself down and dead 2024-09-20T18:42:45.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:44 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:45.521 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:45 smithi174 podman[56938]: 2024-09-20 18:42:45.231821159 +0000 UTC m=+1.500134883 container died b4c31683bf4df27191f62f424710dad31287cdde3bbb6fabe7a1214eea681e99 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, version=8, ceph=True, GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD) 2024-09-20T18:42:45.793 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:45 smithi174 podman[56938]: 2024-09-20 18:42:45.590729114 +0000 UTC m=+1.859042848 container cleanup b4c31683bf4df27191f62f424710dad31287cdde3bbb6fabe7a1214eea681e99 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, name=centos-stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, release=754, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container) 2024-09-20T18:42:45.793 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:45 smithi174 bash[56938]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6 2024-09-20T18:42:46.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:45 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:42:46.215 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:45 smithi174 ceph-mon[49279]: osdmap e129: 8 total, 7 up, 8 in 2024-09-20T18:42:46.216 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:45 smithi174 podman[56963]: 2024-09-20 18:42:45.883927256 +0000 UTC m=+0.647062396 container remove b4c31683bf4df27191f62f424710dad31287cdde3bbb6fabe7a1214eea681e99 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, name=centos-stream, GIT_CLEAN=True, vendor=Red Hat, Inc., RELEASE=HEAD, version=8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-20T18:42:46.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:45 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:42:46.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:45 smithi012 ceph-mon[65348]: osdmap e129: 8 total, 7 up, 8 in 2024-09-20T18:42:46.511 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:46 smithi174 podman[57056]: 2024-09-20 18:42:46.102978724 +0000 UTC m=+0.015820007 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:42:46.511 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:46 smithi174 podman[57056]: 2024-09-20 18:42:46.234985813 +0000 UTC m=+0.147827080 container create 5045d45e5a67eb7a7984c1e62c4a5ea2c564a628c6ec8fc332fc2f0fa0e5a1e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-20T18:42:46.511 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:46 smithi174 podman[57056]: 2024-09-20 18:42:46.495808964 +0000 UTC m=+0.408650230 container init 5045d45e5a67eb7a7984c1e62c4a5ea2c564a628c6ec8fc332fc2f0fa0e5a1e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , ceph=True) 2024-09-20T18:42:46.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:46 smithi174 podman[57056]: 2024-09-20 18:42:46.499054232 +0000 UTC m=+0.411895509 container start 5045d45e5a67eb7a7984c1e62c4a5ea2c564a628c6ec8fc332fc2f0fa0e5a1e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-20T18:42:46.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:46 smithi174 podman[57056]: 2024-09-20 18:42:46.595598494 +0000 UTC m=+0.508439776 container attach 5045d45e5a67eb7a7984c1e62c4a5ea2c564a628c6ec8fc332fc2f0fa0e5a1e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:42:46.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:46 smithi174 podman[57056]: 2024-09-20 18:42:46.690181157 +0000 UTC m=+0.603022424 container died 5045d45e5a67eb7a7984c1e62c4a5ea2c564a628c6ec8fc332fc2f0fa0e5a1e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T18:42:47.214 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:46 smithi174 ceph-mon[49279]: pgmap v227: 97 pgs: 4 peering, 93 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 210 B/s rd, 0 op/s; 0 B/s, 126 objects/s recovering 2024-09-20T18:42:47.214 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:46 smithi174 ceph-mon[49279]: osdmap e130: 8 total, 7 up, 8 in 2024-09-20T18:42:47.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:46 smithi012 ceph-mon[65348]: pgmap v227: 97 pgs: 4 peering, 93 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 210 B/s rd, 0 op/s; 0 B/s, 126 objects/s recovering 2024-09-20T18:42:47.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:46 smithi012 ceph-mon[65348]: osdmap e130: 8 total, 7 up, 8 in 2024-09-20T18:42:47.521 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 podman[57079]: 2024-09-20 18:42:47.198810947 +0000 UTC m=+0.502882228 container remove 5045d45e5a67eb7a7984c1e62c4a5ea2c564a628c6ec8fc332fc2f0fa0e5a1e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-20T18:42:47.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-20T18:42:47.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 systemd[1]: Stopped Ceph osd.6 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:42:47.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.6.service: Consumed 4min 42.470s CPU time. 2024-09-20T18:42:47.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 systemd[1]: Starting Ceph osd.6 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:42:47.909 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:47 smithi174 ceph-mon[49279]: Health check failed: Reduced data availability: 5 pgs peering (PG_AVAILABILITY) 2024-09-20T18:42:47.909 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:47 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 9/444 objects degraded (2.027%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:47.909 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 podman[57180]: 2024-09-20 18:42:47.533230009 +0000 UTC m=+0.015846309 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:42:47.909 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 podman[57180]: 2024-09-20 18:42:47.633290644 +0000 UTC m=+0.115907003 container create 23b3d4c85f18b13664219532eaaa97742bd70c97f01d1ef585473e530026d93e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:42:47.909 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 podman[57180]: 2024-09-20 18:42:47.894239842 +0000 UTC m=+0.376856147 container init 23b3d4c85f18b13664219532eaaa97742bd70c97f01d1ef585473e530026d93e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-20T18:42:48.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:47 smithi012 ceph-mon[65348]: Health check failed: Reduced data availability: 5 pgs peering (PG_AVAILABILITY) 2024-09-20T18:42:48.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:47 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 9/444 objects degraded (2.027%), 3 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:48.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 podman[57180]: 2024-09-20 18:42:47.897668646 +0000 UTC m=+0.380284946 container start 23b3d4c85f18b13664219532eaaa97742bd70c97f01d1ef585473e530026d93e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-20T18:42:48.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:47 smithi174 podman[57180]: 2024-09-20 18:42:47.994139934 +0000 UTC m=+0.476756249 container attach 23b3d4c85f18b13664219532eaaa97742bd70c97f01d1ef585473e530026d93e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-20T18:42:48.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:48 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:48.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:48 smithi174 bash[57180]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:48.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:48 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:48.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:48 smithi174 bash[57180]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:49.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:48 smithi012 ceph-mon[65348]: pgmap v229: 97 pgs: 5 active+undersized, 3 stale+active+clean, 15 peering, 3 active+undersized+degraded, 71 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9/444 objects degraded (2.027%) 2024-09-20T18:42:49.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:48 smithi174 ceph-mon[49279]: pgmap v229: 97 pgs: 5 active+undersized, 3 stale+active+clean, 15 peering, 3 active+undersized+degraded, 71 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9/444 objects degraded (2.027%) 2024-09-20T18:42:51.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:50 smithi012 ceph-mon[65348]: pgmap v230: 97 pgs: 11 active+undersized, 1 stale+active+clean, 15 peering, 8 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32/444 objects degraded (7.207%) 2024-09-20T18:42:51.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:50 smithi174 ceph-mon[49279]: pgmap v230: 97 pgs: 11 active+undersized, 1 stale+active+clean, 15 peering, 8 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 32/444 objects degraded (7.207%) 2024-09-20T18:42:51.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:42:51.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:51.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 bash[57180]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:42:51.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 bash[57180]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:51.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:51.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 bash[57180]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:42:51.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T18:42:51.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 bash[57180]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T18:42:51.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-20T18:42:51.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:50 smithi174 bash[57180]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2024-09-20T18:42:51.628 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 bash[57180]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 bash[57180]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 bash[57180]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 bash[57180]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate[57195]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 bash[57180]: --> ceph-volume lvm activate successful for osd ID: 6 2024-09-20T18:42:51.629 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 podman[57180]: 2024-09-20 18:42:51.309901956 +0000 UTC m=+3.792518258 container died 23b3d4c85f18b13664219532eaaa97742bd70c97f01d1ef585473e530026d93e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-09-20T18:42:52.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:51 smithi012 ceph-mon[65348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs peering) 2024-09-20T18:42:52.235 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:51 smithi174 ceph-mon[49279]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs peering) 2024-09-20T18:42:52.236 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:51 smithi174 podman[57413]: 2024-09-20 18:42:51.897428786 +0000 UTC m=+0.579658532 container remove 23b3d4c85f18b13664219532eaaa97742bd70c97f01d1ef585473e530026d93e (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-20T18:42:52.236 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:52 smithi174 podman[57473]: 2024-09-20 18:42:52.008415155 +0000 UTC m=+0.015709318 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:42:52.236 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:52 smithi174 podman[57473]: 2024-09-20 18:42:52.106447164 +0000 UTC m=+0.113741312 container create b5b9f7c2d0417e6da94c9a3e415a478270d1278bf78953b45b04cce7877c3ab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-20T18:42:52.516 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:52 smithi174 podman[57473]: 2024-09-20 18:42:52.367232644 +0000 UTC m=+0.374526793 container init b5b9f7c2d0417e6da94c9a3e415a478270d1278bf78953b45b04cce7877c3ab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-20T18:42:52.516 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:52 smithi174 podman[57473]: 2024-09-20 18:42:52.371035554 +0000 UTC m=+0.378329713 container start b5b9f7c2d0417e6da94c9a3e415a478270d1278bf78953b45b04cce7877c3ab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:42:52.516 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:52 smithi174 ceph-osd[57491]: -- 172.21.15.174:0/2987437278 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a5bccce1e0 con 0x55a5bccca000 2024-09-20T18:42:52.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:52 smithi174 bash[57473]: b5b9f7c2d0417e6da94c9a3e415a478270d1278bf78953b45b04cce7877c3ab1 2024-09-20T18:42:52.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:52 smithi174 systemd[1]: Started Ceph osd.6 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:42:53.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:52 smithi012 ceph-mon[65348]: pgmap v231: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:53.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:53.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:53.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:52 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:42:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:52 smithi174 ceph-mon[49279]: pgmap v231: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:52 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:42:53.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:42:53 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:42:53.500+0000 7fd3c534b640 -1 Falling back to public interface 2024-09-20T18:42:55.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:54 smithi174 ceph-mon[49279]: pgmap v232: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:55.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:55.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:55.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:54 smithi012 ceph-mon[65348]: pgmap v232: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:42:57.006 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:42:57.166 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:56 smithi174 ceph-mon[49279]: pgmap v233: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:57.166 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.166 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.166 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:56 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 53/444 objects degraded (11.937%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:57.166 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.167 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:56 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:56 smithi012 ceph-mon[65348]: pgmap v233: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:57.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:56 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 53/444 objects degraded (11.937%), 15 pgs degraded (PG_DEGRADED) 2024-09-20T18:42:57.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:56 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:42:57.715 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:42:57.715 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (7m) 109s ago 22m 16.1M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:42:57.715 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (4m) 109s ago 22m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:42:57.715 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (4m) 1s ago 21m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:42:57.715 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (7m) 109s ago 22m 45.4M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:42:57.715 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (16m) 109s ago 16m 41.1M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:42:57.715 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (16m) 1s ago 16m 16.8M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (7m) 109s ago 23m 522M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (6m) 1s ago 21m 457M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (5m) 109s ago 23m 53.1M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (5m) 1s ago 21m 46.2M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (16m) 109s ago 16m 265M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (15m) 1s ago 15m 86.5M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (7m) 109s ago 22m 9.92M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 1s ago 21m 11.0M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (4m) 109s ago 20m 252M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (3m) 109s ago 20m 209M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (2m) 109s ago 19m 193M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:42:57.716 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (115s) 109s ago 19m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:42:57.717 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (77s) 1s ago 19m 219M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:42:57.717 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (41s) 1s ago 18m 276M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b38346191a9d 2024-09-20T18:42:57.717 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (5s) 1s ago 18m 11.2M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b5b9f7c2d041 2024-09-20T18:42:57.717 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (17m) 1s ago 17m 607M 2704M 17.2.0 e1d6a67b021e 160fc1576d20 2024-09-20T18:42:57.717 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (6m) 109s ago 21m 55.3M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:42:58.158 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 6 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:42:58.159 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:42:58.160 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-20T18:42:58.160 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 10 2024-09-20T18:42:58.160 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:42:58.160 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:42:58.523 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:42:58.523 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:42:58.523 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:42:58.523 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "13/23 daemons upgraded", 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:42:58.524 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:42:58.966 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 53/444 objects degraded (11.937%), 15 pgs degraded 2024-09-20T18:42:58.966 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T18:42:58.966 INFO:teuthology.orchestra.run.smithi012.stdout: osd.6 (root=default,host=smithi174) is down 2024-09-20T18:42:58.966 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 53/444 objects degraded (11.937%), 15 pgs degraded 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-20T18:42:58.967 INFO:teuthology.orchestra.run.smithi012.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-20T18:42:58.968 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-20T18:42:58.968 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-20T18:42:59.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:58 smithi012 ceph-mon[65348]: pgmap v234: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:59.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:58 smithi012 ceph-mon[65348]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:59.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:58 smithi012 ceph-mon[65348]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:59.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:58 smithi012 ceph-mon[65348]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:59.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:58 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/3077898666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:42:59.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:58 smithi174 ceph-mon[49279]: pgmap v234: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 53/444 objects degraded (11.937%) 2024-09-20T18:42:59.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:58 smithi174 ceph-mon[49279]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:59.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:58 smithi174 ceph-mon[49279]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:59.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:58 smithi174 ceph-mon[49279]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:42:59.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:58 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/3077898666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:00.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:59 smithi174 ceph-mon[49279]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:00.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:42:59 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/463218269' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:43:00.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:59 smithi012 ceph-mon[65348]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:00.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:42:59 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/463218269' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:43:00.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:43:00.386+0000 7fd3c534b640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: pgmap v235: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 53/444 objects degraded (11.937%) 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:01.160 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:01.161 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:01.161 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: pgmap v235: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 53/444 objects degraded (11.937%) 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:01.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T18:43:01.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:43:01 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:43:01.451+0000 7fd3c534b640 -1 osd.6 128 log_to_monitors true 2024-09-20T18:43:02.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:01 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T18:43:02.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:01 smithi012 ceph-mon[65348]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T18:43:02.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:01 smithi012 ceph-mon[65348]: from='osd.6 [v2:172.21.15.174:6816/1436633295,v1:172.21.15.174:6817/1436633295]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T18:43:02.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:01 smithi012 ceph-mon[65348]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T18:43:02.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:43:01 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:43:01.906+0000 7fd3bd0f5640 -1 osd.6 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:43:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:01 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T18:43:02.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:01 smithi174 ceph-mon[49279]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-20T18:43:02.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:01 smithi174 ceph-mon[49279]: from='osd.6 [v2:172.21.15.174:6816/1436633295,v1:172.21.15.174:6817/1436633295]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T18:43:02.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:01 smithi174 ceph-mon[49279]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-20T18:43:03.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:02 smithi012 ceph-mon[65348]: pgmap v236: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 53/444 objects degraded (11.937%) 2024-09-20T18:43:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:02 smithi012 ceph-mon[65348]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T18:43:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:02 smithi012 ceph-mon[65348]: from='osd.6 [v2:172.21.15.174:6816/1436633295,v1:172.21.15.174:6817/1436633295]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:02 smithi012 ceph-mon[65348]: osdmap e131: 8 total, 7 up, 8 in 2024-09-20T18:43:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:02 smithi012 ceph-mon[65348]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:02 smithi174 ceph-mon[49279]: pgmap v236: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s; 53/444 objects degraded (11.937%) 2024-09-20T18:43:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:02 smithi174 ceph-mon[49279]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-20T18:43:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:02 smithi174 ceph-mon[49279]: from='osd.6 [v2:172.21.15.174:6816/1436633295,v1:172.21.15.174:6817/1436633295]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:02 smithi174 ceph-mon[49279]: osdmap e131: 8 total, 7 up, 8 in 2024-09-20T18:43:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:02 smithi174 ceph-mon[49279]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:03 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:43:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:03 smithi012 ceph-mon[65348]: osd.6 [v2:172.21.15.174:6816/1436633295,v1:172.21.15.174:6817/1436633295] boot 2024-09-20T18:43:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:03 smithi012 ceph-mon[65348]: osdmap e132: 8 total, 8 up, 8 in 2024-09-20T18:43:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T18:43:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:03 smithi012 ceph-mon[65348]: osdmap e133: 8 total, 8 up, 8 in 2024-09-20T18:43:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:03 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:43:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:03 smithi174 ceph-mon[49279]: osd.6 [v2:172.21.15.174:6816/1436633295,v1:172.21.15.174:6817/1436633295] boot 2024-09-20T18:43:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:03 smithi174 ceph-mon[49279]: osdmap e132: 8 total, 8 up, 8 in 2024-09-20T18:43:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-20T18:43:04.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:03 smithi174 ceph-mon[49279]: osdmap e133: 8 total, 8 up, 8 in 2024-09-20T18:43:05.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:04 smithi012 ceph-mon[65348]: pgmap v239: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 53/444 objects degraded (11.937%) 2024-09-20T18:43:05.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:04 smithi174 ceph-mon[49279]: pgmap v239: 97 pgs: 20 active+undersized, 15 active+undersized+degraded, 62 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 53/444 objects degraded (11.937%) 2024-09-20T18:43:06.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:05 smithi012 ceph-mon[65348]: pgmap v241: 97 pgs: 1 peering, 17 active+undersized, 13 active+undersized+degraded, 66 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 47/444 objects degraded (10.586%) 2024-09-20T18:43:06.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:05 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 47/444 objects degraded (10.586%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:06.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:05 smithi174 ceph-mon[49279]: pgmap v241: 97 pgs: 1 peering, 17 active+undersized, 13 active+undersized+degraded, 66 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 47/444 objects degraded (10.586%) 2024-09-20T18:43:06.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:05 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 47/444 objects degraded (10.586%), 13 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:08.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:08 smithi012 ceph-mon[65348]: pgmap v242: 97 pgs: 1 peering, 5 active+undersized, 3 active+undersized+degraded, 88 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 15/444 objects degraded (3.378%) 2024-09-20T18:43:08.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:08 smithi174 ceph-mon[49279]: pgmap v242: 97 pgs: 1 peering, 5 active+undersized, 3 active+undersized+degraded, 88 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s; 15/444 objects degraded (3.378%) 2024-09-20T18:43:09.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:09 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/444 objects degraded (3.378%), 3 pgs degraded) 2024-09-20T18:43:09.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:09 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:43:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:09 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15/444 objects degraded (3.378%), 3 pgs degraded) 2024-09-20T18:43:09.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:09 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:43:10.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:10 smithi012 ceph-mon[65348]: pgmap v243: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 288 B/s rd, 0 op/s 2024-09-20T18:43:10.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:10.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:10 smithi174 ceph-mon[49279]: pgmap v243: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 288 B/s rd, 0 op/s 2024-09-20T18:43:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:12.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:12 smithi012 ceph-mon[65348]: pgmap v244: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-20T18:43:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:12 smithi174 ceph-mon[49279]: pgmap v244: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-20T18:43:14.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:14 smithi012 ceph-mon[65348]: pgmap v245: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:43:15.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:14 smithi174 ceph-mon[49279]: pgmap v245: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s 2024-09-20T18:43:15.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:15 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:15.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:15 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T18:43:15.980 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:15 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:15.980 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:15 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-20T18:43:16.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:16 smithi012 ceph-mon[65348]: pgmap v246: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 184 B/s rd, 0 op/s 2024-09-20T18:43:16.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:16 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T18:43:16.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:16 smithi012 ceph-mon[65348]: Upgrade: osd.7 is safe to restart 2024-09-20T18:43:16.983 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:16 smithi174 ceph-mon[49279]: pgmap v246: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 184 B/s rd, 0 op/s 2024-09-20T18:43:16.983 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:16 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-20T18:43:16.983 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:16 smithi174 ceph-mon[49279]: Upgrade: osd.7 is safe to restart 2024-09-20T18:43:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:17 smithi174 ceph-mon[49279]: pgmap v247: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:43:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:17 smithi174 ceph-mon[49279]: Upgrade: Updating osd.7 2024-09-20T18:43:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T18:43:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:17 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:18.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:17 smithi174 ceph-mon[49279]: Deploying daemon osd.7 on smithi174 2024-09-20T18:43:18.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:17 smithi012 ceph-mon[65348]: pgmap v247: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:43:18.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:17 smithi012 ceph-mon[65348]: Upgrade: Updating osd.7 2024-09-20T18:43:18.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:18.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-20T18:43:18.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:17 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:18.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:17 smithi012 ceph-mon[65348]: Deploying daemon osd.7 on smithi174 2024-09-20T18:43:20.155 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:19 smithi174 ceph-mon[49279]: pgmap v248: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:43:20.155 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:19 smithi174 systemd[1]: Stopping Ceph osd.7 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:43:20.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:19 smithi012 ceph-mon[65348]: pgmap v248: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:43:20.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:20 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[38851]: 2024-09-20T18:43:20.138+0000 7f7eef28c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:43:20.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:20 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[38851]: 2024-09-20T18:43:20.138+0000 7f7eef28c700 -1 osd.7 133 *** Got signal Terminated *** 2024-09-20T18:43:20.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:20 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[38851]: 2024-09-20T18:43:20.138+0000 7f7eef28c700 -1 osd.7 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:43:21.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:20 smithi174 ceph-mon[49279]: osd.7 marked itself down and dead 2024-09-20T18:43:21.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:20 smithi012 ceph-mon[65348]: osd.7 marked itself down and dead 2024-09-20T18:43:21.864 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:21 smithi174 podman[59046]: 2024-09-20 18:43:21.526711098 +0000 UTC m=+1.496887721 container died 160fc1576d20af7701867a810be8a853f77e64ec31f3c47c0326208805a4ceb9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, RELEASE=HEAD, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-09-20T18:43:22.140 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:21 smithi174 podman[59046]: 2024-09-20 18:43:21.947132451 +0000 UTC m=+1.917309065 container cleanup 160fc1576d20af7701867a810be8a853f77e64ec31f3c47c0326208805a4ceb9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, vendor=Red Hat, Inc., architecture=x86_64, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, ceph=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container) 2024-09-20T18:43:22.140 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:21 smithi174 bash[59046]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7 2024-09-20T18:43:22.140 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:21 smithi174 ceph-mon[49279]: pgmap v249: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:43:22.140 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:21 smithi174 ceph-mon[49279]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:43:22.140 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:21 smithi174 ceph-mon[49279]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-20T18:43:22.141 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:21 smithi174 ceph-mon[49279]: osdmap e134: 8 total, 7 up, 8 in 2024-09-20T18:43:22.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:21 smithi012 ceph-mon[65348]: pgmap v249: 97 pgs: 97 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:43:22.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:21 smithi012 ceph-mon[65348]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-20T18:43:22.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:21 smithi012 ceph-mon[65348]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-20T18:43:22.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:21 smithi012 ceph-mon[65348]: osdmap e134: 8 total, 7 up, 8 in 2024-09-20T18:43:22.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:22 smithi174 podman[59090]: 2024-09-20 18:43:22.124702747 +0000 UTC m=+0.599719822 container remove 160fc1576d20af7701867a810be8a853f77e64ec31f3c47c0326208805a4ceb9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, distribution-scope=public, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, release=754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_BRANCH=HEAD) 2024-09-20T18:43:22.853 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:22 smithi174 podman[59182]: 2024-09-20 18:43:22.55176939 +0000 UTC m=+0.105198442 container create 4f713dd8dd9f50eb8b4541e9740ca0d2926b5cb8442bd10fdfc9abce0b29da88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-20T18:43:22.854 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:22 smithi174 podman[59182]: 2024-09-20 18:43:22.46244235 +0000 UTC m=+0.015871410 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:43:22.854 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:22 smithi174 podman[59182]: 2024-09-20 18:43:22.755478485 +0000 UTC m=+0.308907543 container init 4f713dd8dd9f50eb8b4541e9740ca0d2926b5cb8442bd10fdfc9abce0b29da88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-20T18:43:22.854 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:22 smithi174 podman[59182]: 2024-09-20 18:43:22.759309839 +0000 UTC m=+0.312738890 container start 4f713dd8dd9f50eb8b4541e9740ca0d2926b5cb8442bd10fdfc9abce0b29da88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:43:22.854 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:22 smithi174 podman[59182]: 2024-09-20 18:43:22.838416333 +0000 UTC m=+0.391845396 container attach 4f713dd8dd9f50eb8b4541e9740ca0d2926b5cb8442bd10fdfc9abce0b29da88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True) 2024-09-20T18:43:23.251 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:23 smithi174 ceph-mon[49279]: osdmap e135: 8 total, 7 up, 8 in 2024-09-20T18:43:23.251 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:22 smithi174 podman[59182]: 2024-09-20 18:43:22.945485221 +0000 UTC m=+0.498914282 container died 4f713dd8dd9f50eb8b4541e9740ca0d2926b5cb8442bd10fdfc9abce0b29da88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:43:23.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:23 smithi012 ceph-mon[65348]: osdmap e135: 8 total, 7 up, 8 in 2024-09-20T18:43:23.962 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:23 smithi174 podman[59206]: 2024-09-20 18:43:23.510932851 +0000 UTC m=+0.555187306 container remove 4f713dd8dd9f50eb8b4541e9740ca0d2926b5cb8442bd10fdfc9abce0b29da88 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=) 2024-09-20T18:43:23.962 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:23 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-20T18:43:23.962 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:23 smithi174 systemd[1]: Stopped Ceph osd.7 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:43:23.962 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:23 smithi174 systemd[1]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.7.service: Consumed 6min 6.406s CPU time. 2024-09-20T18:43:23.962 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:23 smithi174 systemd[1]: Starting Ceph osd.7 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:43:24.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:24 smithi174 ceph-mon[49279]: pgmap v252: 97 pgs: 4 active+undersized, 15 peering, 7 stale+active+clean, 6 active+undersized+degraded, 65 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 24/444 objects degraded (5.405%) 2024-09-20T18:43:24.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:24 smithi174 ceph-mon[49279]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-09-20T18:43:24.257 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:24 smithi174 ceph-mon[49279]: Health check failed: Degraded data redundancy: 24/444 objects degraded (5.405%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:24.257 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:23 smithi174 podman[59304]: 2024-09-20 18:43:23.849139364 +0000 UTC m=+0.015978256 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:43:24.258 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:23 smithi174 podman[59304]: 2024-09-20 18:43:23.959496893 +0000 UTC m=+0.126335780 container create 2cb3415295c951027f6b0c6174075bd66d24d9836cde27dff1af2904f7a7f5d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-20T18:43:24.258 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:24 smithi174 podman[59304]: 2024-09-20 18:43:24.167183754 +0000 UTC m=+0.334022641 container init 2cb3415295c951027f6b0c6174075bd66d24d9836cde27dff1af2904f7a7f5d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-20T18:43:24.258 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:24 smithi174 podman[59304]: 2024-09-20 18:43:24.170567336 +0000 UTC m=+0.337406223 container start 2cb3415295c951027f6b0c6174075bd66d24d9836cde27dff1af2904f7a7f5d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-20T18:43:24.258 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:24 smithi174 podman[59304]: 2024-09-20 18:43:24.242044845 +0000 UTC m=+0.408883732 container attach 2cb3415295c951027f6b0c6174075bd66d24d9836cde27dff1af2904f7a7f5d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2) 2024-09-20T18:43:24.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:24 smithi012 ceph-mon[65348]: pgmap v252: 97 pgs: 4 active+undersized, 15 peering, 7 stale+active+clean, 6 active+undersized+degraded, 65 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 24/444 objects degraded (5.405%) 2024-09-20T18:43:24.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:24 smithi012 ceph-mon[65348]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-09-20T18:43:24.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:24 smithi012 ceph-mon[65348]: Health check failed: Degraded data redundancy: 24/444 objects degraded (5.405%), 6 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:24.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:24.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:24 smithi174 bash[59304]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:24.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:24.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:24 smithi174 bash[59304]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:25.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:25.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:25 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:26.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:26.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:25 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:26.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:26 smithi012 ceph-mon[65348]: pgmap v253: 97 pgs: 5 active+undersized, 15 peering, 6 stale+active+clean, 7 active+undersized+degraded, 64 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 30/444 objects degraded (6.757%) 2024-09-20T18:43:27.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:26 smithi174 ceph-mon[49279]: pgmap v253: 97 pgs: 5 active+undersized, 15 peering, 6 stale+active+clean, 7 active+undersized+degraded, 64 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 30/444 objects degraded (6.757%) 2024-09-20T18:43:27.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:43:27.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:27.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: --> Failed to activate via raw: did not find any matching OSD to activate 2024-09-20T18:43:27.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:27.521 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:27.522 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-20T18:43:27.522 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T18:43:27.522 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T18:43:27.522 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-20T18:43:27.522 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2024-09-20T18:43:27.885 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-mon[49279]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-09-20T18:43:27.885 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-20T18:43:27.885 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-20T18:43:27.885 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-20T18:43:27.885 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2024-09-20T18:43:27.885 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T18:43:27.885 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-20T18:43:27.885 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T18:43:27.886 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-20T18:43:27.886 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate[59319]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-20T18:43:27.886 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 bash[59304]: --> ceph-volume lvm activate successful for osd ID: 7 2024-09-20T18:43:27.886 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:27 smithi174 podman[59304]: 2024-09-20 18:43:27.587926743 +0000 UTC m=+3.754765641 container died 2cb3415295c951027f6b0c6174075bd66d24d9836cde27dff1af2904f7a7f5d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-20T18:43:27.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:27 smithi012 ceph-mon[65348]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-09-20T18:43:28.614 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 podman[59530]: 2024-09-20 18:43:28.270686313 +0000 UTC m=+0.675663520 container remove 2cb3415295c951027f6b0c6174075bd66d24d9836cde27dff1af2904f7a7f5d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-activate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-20T18:43:28.614 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 podman[59593]: 2024-09-20 18:43:28.381493867 +0000 UTC m=+0.015697339 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:43:28.614 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 podman[59593]: 2024-09-20 18:43:28.496482884 +0000 UTC m=+0.130686387 container create cc57872beb577df28d72d92cb92ee1e15a6848b05b3b859f1e79253f2b386e79 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:43:28.975 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:28 smithi012 ceph-mon[65348]: pgmap v254: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:29.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:28 smithi174 ceph-mon[49279]: pgmap v254: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:29.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 podman[59593]: 2024-09-20 18:43:28.691994864 +0000 UTC m=+0.326198324 container init cc57872beb577df28d72d92cb92ee1e15a6848b05b3b859f1e79253f2b386e79 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-20T18:43:29.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 podman[59593]: 2024-09-20 18:43:28.695416135 +0000 UTC m=+0.329619594 container start cc57872beb577df28d72d92cb92ee1e15a6848b05b3b859f1e79253f2b386e79 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:43:29.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 ceph-osd[59612]: -- 172.21.15.174:0/253760492 <== mon.1 v2:172.21.15.174:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55622dc9c1e0 con 0x55622dc98000 2024-09-20T18:43:29.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 bash[59593]: cc57872beb577df28d72d92cb92ee1e15a6848b05b3b859f1e79253f2b386e79 2024-09-20T18:43:29.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:28 smithi174 systemd[1]: Started Ceph osd.7 for c01c9b48-777c-11ef-baf5-efdc52797490. 2024-09-20T18:43:29.358 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:43:30.074 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:43:30.074 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (8m) 2m ago 22m 16.1M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:43:30.074 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (5m) 2m ago 22m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (5m) 34s ago 22m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (8m) 2m ago 22m 45.4M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (16m) 2m ago 16m 41.1M - 17.2.0 e1d6a67b021e 94dddbf9db78 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (16m) 34s ago 16m 16.8M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (7m) 2m ago 24m 522M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (6m) 34s ago 22m 457M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (6m) 2m ago 24m 53.1M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (6m) 34s ago 22m 46.2M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (16m) 2m ago 16m 265M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (16m) 34s ago 16m 86.5M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (8m) 2m ago 22m 9.92M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (7m) 34s ago 22m 11.0M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:43:30.075 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (4m) 2m ago 21m 252M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (3m) 2m ago 20m 209M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (3m) 2m ago 20m 193M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (2m) 2m ago 19m 12.1M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (109s) 34s ago 19m 219M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (73s) 34s ago 19m 276M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b38346191a9d 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (38s) 34s ago 18m 11.2M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b5b9f7c2d041 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 starting - - - 2704M 2024-09-20T18:43:30.076 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (6m) 2m ago 21m 55.3M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:43:30.123 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:30.123 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:30.123 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:29 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:30.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:30.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:30.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:29 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:30.389 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:30 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:43:30.106+0000 7f380ddc7640 -1 Falling back to public interface 2024-09-20T18:43:30.513 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:43:30.513 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:43:30.513 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:43:30.513 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 7 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:43:30.514 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:43:30.515 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-20T18:43:30.515 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 11 2024-09-20T18:43:30.515 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:43:30.515 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:43:30.879 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:43:30.879 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:43:30.879 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:43:30.879 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:43:30.879 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:43:30.879 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:43:30.880 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:43:30.880 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:43:30.880 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:43:30.880 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "13/23 daemons upgraded", 2024-09-20T18:43:30.880 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading osd daemons", 2024-09-20T18:43:30.880 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:43:30.880 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:43:31.117 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:30 smithi174 ceph-mon[49279]: pgmap v255: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:31.117 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:30 smithi174 ceph-mon[49279]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:31.117 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:30 smithi174 ceph-mon[49279]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:31.118 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:30 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/767022641' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:31.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:30 smithi012 ceph-mon[65348]: pgmap v255: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:31.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:30 smithi012 ceph-mon[65348]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:31.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:30 smithi012 ceph-mon[65348]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:31.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:30 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/767022641' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:31.315 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 78/444 objects degraded (17.568%), 22 pgs degraded 2024-09-20T18:43:31.315 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-20T18:43:31.315 INFO:teuthology.orchestra.run.smithi012.stdout: osd.7 (root=default,host=smithi174) is down 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 78/444 objects degraded (17.568%), 22 pgs degraded 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-20T18:43:31.316 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.0 is active+undersized+degraded, acting [5,0] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-20T18:43:31.317 INFO:teuthology.orchestra.run.smithi012.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-09-20T18:43:32.013 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:31 smithi174 ceph-mon[49279]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:32.014 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:31 smithi174 ceph-mon[49279]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:32.014 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:31 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 78/444 objects degraded (17.568%), 22 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:32.014 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:31 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/1992689327' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:43:32.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:31 smithi012 ceph-mon[65348]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:32.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:31 smithi012 ceph-mon[65348]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:43:32.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:31 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 78/444 objects degraded (17.568%), 22 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:32.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:31 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/1992689327' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:43:33.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:32 smithi174 ceph-mon[49279]: pgmap v256: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:33.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:33.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:33.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:33.164 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:33.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:32 smithi012 ceph-mon[65348]: pgmap v256: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:33.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:33.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:33.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:33.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:35.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:34 smithi012 ceph-mon[65348]: pgmap v257: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:35.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:34 smithi174 ceph-mon[49279]: pgmap v257: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:37.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: pgmap v258: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:37.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-20T18:43:37.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-20T18:43:37.228 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:36 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-20T18:43:37.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: pgmap v258: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-20T18:43:37.273 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-20T18:43:37.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-20T18:43:37.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-20T18:43:37.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-20T18:43:37.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:36 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-20T18:43:38.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:37 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all osd 2024-09-20T18:43:38.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:37 smithi012 ceph-mon[65348]: Upgrade: Setting require_osd_release to 19 squid 2024-09-20T18:43:38.210 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:37 smithi012 ceph-mon[65348]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-20T18:43:38.210 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:37 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-20T18:43:38.210 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:37 smithi012 ceph-mon[65348]: osdmap e136: 8 total, 7 up, 8 in 2024-09-20T18:43:38.210 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:37 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi012.jjsnmc"]} : dispatch 2024-09-20T18:43:38.271 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:43:37.936+0000 7f380ddc7640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-20T18:43:38.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:37 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all osd 2024-09-20T18:43:38.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:37 smithi174 ceph-mon[49279]: Upgrade: Setting require_osd_release to 19 squid 2024-09-20T18:43:38.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:37 smithi174 ceph-mon[49279]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-20T18:43:38.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:37 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-20T18:43:38.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:37 smithi174 ceph-mon[49279]: osdmap e136: 8 total, 7 up, 8 in 2024-09-20T18:43:38.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:37 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi012.jjsnmc"]} : dispatch 2024-09-20T18:43:39.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:38 smithi012 ceph-mon[65348]: pgmap v259: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:39.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:38 smithi012 ceph-mon[65348]: Upgrade: It appears safe to stop mds.foofs.smithi012.jjsnmc 2024-09-20T18:43:39.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:38 smithi174 ceph-mon[49279]: pgmap v259: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:39.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:38 smithi174 ceph-mon[49279]: Upgrade: It appears safe to stop mds.foofs.smithi012.jjsnmc 2024-09-20T18:43:39.771 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:43:39.341+0000 7f380ddc7640 -1 osd.7 133 log_to_monitors true 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: pgmap v261: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 78/444 objects degraded (17.568%) 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: Upgrade: Updating mds.foofs.smithi012.jjsnmc 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: Deploying daemon mds.foofs.smithi012.jjsnmc on smithi012 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: from='osd.7 [v2:172.21.15.174:6824/1588183647,v1:172.21.15.174:6825/1588183647]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T18:43:40.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:40.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:40 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:40.539 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: pgmap v261: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 0 op/s; 78/444 objects degraded (17.568%) 2024-09-20T18:43:40.539 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: Upgrade: Updating mds.foofs.smithi012.jjsnmc 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi012.jjsnmc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: Deploying daemon mds.foofs.smithi012.jjsnmc on smithi012 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: from='osd.7 [v2:172.21.15.174:6824/1588183647,v1:172.21.15.174:6825/1588183647]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:40.540 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:41.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:43:40 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:43:40.602+0000 7f3801b69640 -1 osd.7 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-20T18:43:41.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:41 smithi012 ceph-mon[65348]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T18:43:41.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:41 smithi012 ceph-mon[65348]: osdmap e137: 8 total, 7 up, 8 in 2024-09-20T18:43:41.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:41 smithi012 ceph-mon[65348]: from='osd.7 [v2:172.21.15.174:6824/1588183647,v1:172.21.15.174:6825/1588183647]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:41.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:41 smithi012 ceph-mon[65348]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:41.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:41 smithi012 ceph-mon[65348]: from='osd.7 ' entity='osd.7' 2024-09-20T18:43:41.427 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:41 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[65344]: 2024-09-20T18:43:41.221+0000 7ff075548640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:43:41.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:41 smithi174 ceph-mon[49279]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-20T18:43:41.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:41 smithi174 ceph-mon[49279]: osdmap e137: 8 total, 7 up, 8 in 2024-09-20T18:43:41.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:41 smithi174 ceph-mon[49279]: from='osd.7 [v2:172.21.15.174:6824/1588183647,v1:172.21.15.174:6825/1588183647]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:41.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:41 smithi174 ceph-mon[49279]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi174", "root=default"]} : dispatch 2024-09-20T18:43:41.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:41 smithi174 ceph-mon[49279]: from='osd.7 ' entity='osd.7' 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: pgmap v263: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: osd.7 [v2:172.21.15.174:6824/1588183647,v1:172.21.15.174:6825/1588183647] boot 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: osdmap e138: 8 total, 8 up, 8 in 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: Standby daemon mds.foofs.smithi174.zwzgjx assigned to filesystem foofs as rank 0 2024-09-20T18:43:42.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T18:43:42.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T18:43:42.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T18:43:42.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T18:43:42.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:42 smithi012 ceph-mon[65348]: fsmap foofs:1/1 {0=foofs.smithi174.zwzgjx=up:replay} 2024-09-20T18:43:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: pgmap v263: 97 pgs: 19 active+undersized, 22 active+undersized+degraded, 56 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 78/444 objects degraded (17.568%) 2024-09-20T18:43:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-20T18:43:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T18:43:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:43:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: osd.7 [v2:172.21.15.174:6824/1588183647,v1:172.21.15.174:6825/1588183647] boot 2024-09-20T18:43:42.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: osdmap e138: 8 total, 8 up, 8 in 2024-09-20T18:43:42.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: Standby daemon mds.foofs.smithi174.zwzgjx assigned to filesystem foofs as rank 0 2024-09-20T18:43:42.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T18:43:42.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T18:43:42.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T18:43:42.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-20T18:43:42.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:42 smithi174 ceph-mon[49279]: fsmap foofs:1/1 {0=foofs.smithi174.zwzgjx=up:replay} 2024-09-20T18:43:43.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:43 smithi174 ceph-mon[49279]: osdmap e139: 8 total, 8 up, 8 in 2024-09-20T18:43:43.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:43 smithi012 ceph-mon[65348]: osdmap e139: 8 total, 8 up, 8 in 2024-09-20T18:43:44.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:44 smithi174 ceph-mon[49279]: pgmap v266: 97 pgs: 12 peering, 11 active+undersized, 9 active+undersized+degraded, 65 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4 op/s; 32/444 objects degraded (7.207%) 2024-09-20T18:43:44.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:44 smithi174 ceph-mon[49279]: Health check update: Degraded data redundancy: 32/444 objects degraded (7.207%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:44.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:44 smithi174 ceph-mon[49279]: reconnect by client.24345 172.21.15.174:0/397365867 after 0.000999987 2024-09-20T18:43:44.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:44 smithi174 ceph-mon[49279]: reconnect by client.24329 172.21.15.12:0/862280390 after 0.000999987 2024-09-20T18:43:44.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:44 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:reconnect 2024-09-20T18:43:44.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:44 smithi174 ceph-mon[49279]: fsmap foofs:1/1 {0=foofs.smithi174.zwzgjx=up:reconnect} 2024-09-20T18:43:44.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:44 smithi012 ceph-mon[65348]: pgmap v266: 97 pgs: 12 peering, 11 active+undersized, 9 active+undersized+degraded, 65 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4 op/s; 32/444 objects degraded (7.207%) 2024-09-20T18:43:44.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:44 smithi012 ceph-mon[65348]: Health check update: Degraded data redundancy: 32/444 objects degraded (7.207%), 9 pgs degraded (PG_DEGRADED) 2024-09-20T18:43:44.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:44 smithi012 ceph-mon[65348]: reconnect by client.24345 172.21.15.174:0/397365867 after 0.000999987 2024-09-20T18:43:44.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:44 smithi012 ceph-mon[65348]: reconnect by client.24329 172.21.15.12:0/862280390 after 0.000999987 2024-09-20T18:43:44.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:44 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:reconnect 2024-09-20T18:43:44.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:44 smithi012 ceph-mon[65348]: fsmap foofs:1/1 {0=foofs.smithi174.zwzgjx=up:reconnect} 2024-09-20T18:43:45.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:45 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:rejoin 2024-09-20T18:43:45.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:45 smithi174 ceph-mon[49279]: fsmap foofs:1/1 {0=foofs.smithi174.zwzgjx=up:rejoin} 2024-09-20T18:43:45.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:45 smithi174 ceph-mon[49279]: daemon mds.foofs.smithi174.zwzgjx is now active in filesystem foofs as rank 0 2024-09-20T18:43:45.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:45 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:rejoin 2024-09-20T18:43:45.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:45 smithi012 ceph-mon[65348]: fsmap foofs:1/1 {0=foofs.smithi174.zwzgjx=up:rejoin} 2024-09-20T18:43:45.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:45 smithi012 ceph-mon[65348]: daemon mds.foofs.smithi174.zwzgjx is now active in filesystem foofs as rank 0 2024-09-20T18:43:46.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:46 smithi174 ceph-mon[49279]: pgmap v267: 97 pgs: 12 peering, 11 active+undersized, 8 active+undersized+degraded, 66 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 5 op/s; 26/444 objects degraded (5.856%) 2024-09-20T18:43:46.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:46 smithi174 ceph-mon[49279]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T18:43:46.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:46 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:active 2024-09-20T18:43:46.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:46 smithi174 ceph-mon[49279]: fsmap foofs:1 {0=foofs.smithi174.zwzgjx=up:active} 2024-09-20T18:43:46.725 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:46 smithi012 ceph-mon[65348]: pgmap v267: 97 pgs: 12 peering, 11 active+undersized, 8 active+undersized+degraded, 66 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 5 op/s; 26/444 objects degraded (5.856%) 2024-09-20T18:43:46.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:46 smithi012 ceph-mon[65348]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T18:43:46.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:46 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.174:6832/3662915709,v1:172.21.15.174:6833/3662915709] up:active 2024-09-20T18:43:46.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:46 smithi012 ceph-mon[65348]: fsmap foofs:1 {0=foofs.smithi174.zwzgjx=up:active} 2024-09-20T18:43:47.388 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:47 smithi012 ceph-mon[65348]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/444 objects degraded (5.856%), 8 pgs degraded) 2024-09-20T18:43:47.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:47 smithi174 ceph-mon[49279]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/444 objects degraded (5.856%), 8 pgs degraded) 2024-09-20T18:43:48.568 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:48 smithi012 ceph-mon[65348]: pgmap v268: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 35 MiB/s rd, 149 B/s wr, 12 op/s 2024-09-20T18:43:48.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:48 smithi174 ceph-mon[49279]: pgmap v268: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 35 MiB/s rd, 149 B/s wr, 12 op/s 2024-09-20T18:43:50.164 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:50.164 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:50.165 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:50.165 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T18:43:50.165 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:43:50.165 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:boot 2024-09-20T18:43:50.165 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: fsmap foofs:1 {0=foofs.smithi174.zwzgjx=up:active} 1 up:standby 2024-09-20T18:43:50.165 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:49 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"} : dispatch 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:boot 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: fsmap foofs:1 {0=foofs.smithi174.zwzgjx=up:active} 1 up:standby 2024-09-20T18:43:50.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:49 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi012.jjsnmc"} : dispatch 2024-09-20T18:43:51.173 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:50 smithi174 ceph-mon[49279]: pgmap v269: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 255 B/s wr, 13 op/s 2024-09-20T18:43:51.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:50 smithi012 ceph-mon[65348]: pgmap v269: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 255 B/s wr, 13 op/s 2024-09-20T18:43:53.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:52 smithi174 ceph-mon[49279]: pgmap v270: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 209 B/s wr, 11 op/s 2024-09-20T18:43:53.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:52 smithi012 ceph-mon[65348]: pgmap v270: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 209 B/s wr, 11 op/s 2024-09-20T18:43:54.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:54 smithi012 ceph-mon[65348]: pgmap v271: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 6.7 KiB/s wr, 13 op/s 2024-09-20T18:43:54.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:54.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:54.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:54.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:55.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:54 smithi174 ceph-mon[49279]: pgmap v271: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 MiB/s rd, 6.7 KiB/s wr, 13 op/s 2024-09-20T18:43:55.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:55.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:55.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:55.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:55.902 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:55 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:55.902 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:55 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:55.903 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:55 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:56.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:55 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:56.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:55 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:43:56.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:55 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:56.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:56 smithi012 ceph-mon[65348]: pgmap v272: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 6.0 KiB/s wr, 10 op/s 2024-09-20T18:43:57.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:56 smithi174 ceph-mon[49279]: pgmap v272: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 6.0 KiB/s wr, 10 op/s 2024-09-20T18:43:58.498 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: pgmap v273: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.0 KiB/s wr, 9 op/s 2024-09-20T18:43:58.498 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: Detected new or changed devices on smithi012 2024-09-20T18:43:58.498 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.498 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.498 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi174.zwzgjx"]} : dispatch 2024-09-20T18:43:58.499 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:43:58 smithi174 ceph-mon[49279]: Upgrade: It appears safe to stop mds.foofs.smithi174.zwzgjx 2024-09-20T18:43:58.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: pgmap v273: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.0 KiB/s wr, 9 op/s 2024-09-20T18:43:58.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: Detected new or changed devices on smithi012 2024-09-20T18:43:58.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi174.zwzgjx"]} : dispatch 2024-09-20T18:43:58.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:43:58 smithi012 ceph-mon[65348]: Upgrade: It appears safe to stop mds.foofs.smithi174.zwzgjx 2024-09-20T18:44:00.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:00 smithi012 ceph-mon[65348]: pgmap v274: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 421 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-20T18:44:00.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:00 smithi012 ceph-mon[65348]: Upgrade: Updating mds.foofs.smithi174.zwzgjx 2024-09-20T18:44:00.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:00.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:44:00.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:00 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:00.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:00 smithi012 ceph-mon[65348]: Deploying daemon mds.foofs.smithi174.zwzgjx on smithi174 2024-09-20T18:44:00.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-mon[49279]: pgmap v274: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 421 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-20T18:44:00.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-mon[49279]: Upgrade: Updating mds.foofs.smithi174.zwzgjx 2024-09-20T18:44:00.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:00.504 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi174.zwzgjx", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-20T18:44:00.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:00.505 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-mon[49279]: Deploying daemon mds.foofs.smithi174.zwzgjx on smithi174 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:00 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[65344]: 2024-09-20T18:44:00.944+0000 7ff075548640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: osdmap e140: 8 total, 8 up, 8 in 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: Standby daemon mds.foofs.smithi012.jjsnmc assigned to filesystem foofs as rank 0 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T18:44:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:01 smithi012 ceph-mon[65348]: fsmap foofs:1/1 {0=foofs.smithi012.jjsnmc=up:replay} 2024-09-20T18:44:01.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:44:00.966+0000 7fd3a68ae640 -1 bdev(0x55a5bcd28c00 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 3 2024-09-20T18:44:01.271 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:44:00 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:44:00.965+0000 7f37ec324640 -1 bdev(0x55622dcf6c00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 2 2024-09-20T18:44:01.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-20T18:44:01.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-20T18:44:01.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: osdmap e140: 8 total, 8 up, 8 in 2024-09-20T18:44:01.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: Standby daemon mds.foofs.smithi012.jjsnmc assigned to filesystem foofs as rank 0 2024-09-20T18:44:01.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-20T18:44:01.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-20T18:44:01.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-20T18:44:01.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:01 smithi174 ceph-mon[49279]: fsmap foofs:1/1 {0=foofs.smithi012.jjsnmc=up:replay} 2024-09-20T18:44:01.741 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:44:02.477 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:44:02.477 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (9m) 8s ago 23m 18.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:44:02.477 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (5m) 8s ago 23m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:44:02.477 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (5m) 30s ago 22m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:44:02.477 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (8m) 8s ago 23m 53.0M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:44:02.477 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (14s) 8s ago 17m 11.5M - 19.3.0-5099-g88deadb8 441f9a3fe515 87f9941648bc 2024-09-20T18:44:02.477 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (17m) 30s ago 17m 16.9M - 17.2.0 e1d6a67b021e 7ab08b9827fd 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (8m) 8s ago 24m 526M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (7m) 30s ago 22m 458M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (7m) 8s ago 24m 59.9M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (6m) 30s ago 22m 47.4M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (17m) 8s ago 17m 251M - 4.0 e1d6a67b021e cbb3bf5fdb2f 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (17m) 30s ago 17m 86.5M - 4.0 e1d6a67b021e 37be99feddc4 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (8m) 8s ago 23m 9.96M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 30s ago 22m 12.0M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (5m) 8s ago 21m 288M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (4m) 8s ago 21m 245M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (3m) 8s ago 20m 241M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (3m) 8s ago 20m 307M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (2m) 30s ago 20m 219M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:44:02.478 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (106s) 30s ago 19m 276M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b38346191a9d 2024-09-20T18:44:02.479 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (70s) 30s ago 19m 208M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b5b9f7c2d041 2024-09-20T18:44:02.479 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (34s) 30s ago 18m 13.3M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 cc57872beb57 2024-09-20T18:44:02.479 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (7m) 8s ago 22m 57.7M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:44:02.912 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:44:02.912 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:44:02.912 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:44:02.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:02.913 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:44:02.913 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:44:02.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:02.913 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:44:02.913 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 8 2024-09-20T18:44:02.913 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:02.914 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:44:02.914 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 1 2024-09-20T18:44:02.914 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:02.914 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:44:02.914 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 13 2024-09-20T18:44:02.914 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:44:02.914 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:44:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:02 smithi012 ceph-mon[65348]: pgmap v276: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 8.5 KiB/s wr, 3 op/s 2024-09-20T18:44:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:02 smithi012 ceph-mon[65348]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:02 smithi012 ceph-mon[65348]: reconnect by client.24329 172.21.15.12:0/862280390 after 0.000999992 2024-09-20T18:44:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:02 smithi012 ceph-mon[65348]: reconnect by client.24345 172.21.15.174:0/397365867 after 0.000999992 2024-09-20T18:44:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:02 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:reconnect 2024-09-20T18:44:03.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:02 smithi012 ceph-mon[65348]: fsmap foofs:1/1 {0=foofs.smithi012.jjsnmc=up:reconnect} 2024-09-20T18:44:03.227 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:02 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/4147027307' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:02 smithi174 ceph-mon[49279]: pgmap v276: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 204 B/s rd, 8.5 KiB/s wr, 3 op/s 2024-09-20T18:44:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:02 smithi174 ceph-mon[49279]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:02 smithi174 ceph-mon[49279]: reconnect by client.24329 172.21.15.12:0/862280390 after 0.000999992 2024-09-20T18:44:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:02 smithi174 ceph-mon[49279]: reconnect by client.24345 172.21.15.174:0/397365867 after 0.000999992 2024-09-20T18:44:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:02 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:reconnect 2024-09-20T18:44:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:02 smithi174 ceph-mon[49279]: fsmap foofs:1/1 {0=foofs.smithi012.jjsnmc=up:reconnect} 2024-09-20T18:44:03.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:02 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/4147027307' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:03.282 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:44:03.282 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:44:03.282 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:44:03.282 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "crash", 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "osd", 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr" 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "15/23 daemons upgraded", 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading mds daemons", 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:44:03.283 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:44:03.721 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_WARN 1 filesystem is degraded; insufficient standby MDS daemons available 2024-09-20T18:44:03.721 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-09-20T18:44:03.721 INFO:teuthology.orchestra.run.smithi012.stdout: fs foofs is degraded 2024-09-20T18:44:03.721 INFO:teuthology.orchestra.run.smithi012.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-20T18:44:03.721 INFO:teuthology.orchestra.run.smithi012.stdout: have 0; want 1 more 2024-09-20T18:44:04.274 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:03 smithi174 ceph-mon[49279]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:04.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:03 smithi174 ceph-mon[49279]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:04.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:03 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:rejoin 2024-09-20T18:44:04.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:03 smithi174 ceph-mon[49279]: fsmap foofs:1/1 {0=foofs.smithi012.jjsnmc=up:rejoin} 2024-09-20T18:44:04.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:03 smithi174 ceph-mon[49279]: daemon mds.foofs.smithi012.jjsnmc is now active in filesystem foofs as rank 0 2024-09-20T18:44:04.275 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:03 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/1090775200' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:44:04.322 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:03 smithi012 ceph-mon[65348]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:04.322 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:03 smithi012 ceph-mon[65348]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:04.322 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:03 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:rejoin 2024-09-20T18:44:04.322 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:03 smithi012 ceph-mon[65348]: fsmap foofs:1/1 {0=foofs.smithi012.jjsnmc=up:rejoin} 2024-09-20T18:44:04.322 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:03 smithi012 ceph-mon[65348]: daemon mds.foofs.smithi012.jjsnmc is now active in filesystem foofs as rank 0 2024-09-20T18:44:04.322 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:03 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/1090775200' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:44:05.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:04 smithi174 ceph-mon[49279]: pgmap v277: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 KiB/s wr, 5 op/s 2024-09-20T18:44:05.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:04 smithi174 ceph-mon[49279]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:05.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:04 smithi174 ceph-mon[49279]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T18:44:05.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:04 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:active 2024-09-20T18:44:05.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:04 smithi174 ceph-mon[49279]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 2024-09-20T18:44:05.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:04 smithi012 ceph-mon[65348]: pgmap v277: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 KiB/s wr, 5 op/s 2024-09-20T18:44:05.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:04 smithi012 ceph-mon[65348]: from='client.34456 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:05.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:04 smithi012 ceph-mon[65348]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-20T18:44:05.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:04 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829] up:active 2024-09-20T18:44:05.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:04 smithi012 ceph-mon[65348]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 2024-09-20T18:44:06.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:05 smithi174 ceph-mon[49279]: pgmap v278: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-09-20T18:44:06.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:05 smithi012 ceph-mon[65348]: pgmap v278: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 1.7 KiB/s wr, 6 op/s 2024-09-20T18:44:08.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:08 smithi012 ceph-mon[65348]: pgmap v279: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.8 KiB/s wr, 10 op/s 2024-09-20T18:44:08.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:08 smithi174 ceph-mon[49279]: pgmap v279: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 1.8 KiB/s wr, 10 op/s 2024-09-20T18:44:10.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:10 smithi012 ceph-mon[65348]: pgmap v280: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 409 B/s wr, 11 op/s 2024-09-20T18:44:10.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:10.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:10 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:11.011 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:10 smithi174 ceph-mon[49279]: pgmap v280: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 409 B/s wr, 11 op/s 2024-09-20T18:44:11.011 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:11.011 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:10 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: pgmap v281: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 407 B/s wr, 11 op/s 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: Cluster is now healthy 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: mds.? [v2:172.21.15.174:6832/2179873551,v1:172.21.15.174:6833/2179873551] up:boot 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:44:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:12 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"} : dispatch 2024-09-20T18:44:13.269 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: pgmap v281: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 407 B/s wr, 11 op/s 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: Cluster is now healthy 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: mds.? [v2:172.21.15.174:6832/2179873551,v1:172.21.15.174:6833/2179873551] up:boot 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: fsmap foofs:1 {0=foofs.smithi012.jjsnmc=up:active} 1 up:standby 2024-09-20T18:44:13.270 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:12 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "mds metadata", "who": "foofs.smithi174.zwzgjx"} : dispatch 2024-09-20T18:44:15.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:14 smithi012 ceph-mon[65348]: pgmap v282: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-09-20T18:44:15.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:14 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:15.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:14 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:15.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:14 smithi174 ceph-mon[49279]: pgmap v282: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 MiB/s rd, 7.4 KiB/s wr, 12 op/s 2024-09-20T18:44:15.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:14 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:15.228 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:14 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:16.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:16 smithi012 ceph-mon[65348]: pgmap v283: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-20T18:44:16.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:16 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:16.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:16 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:16.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:16 smithi174 ceph-mon[49279]: pgmap v283: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-20T18:44:16.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:16 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:16.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:16 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:18.348 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:18 smithi174 ceph-mon[49279]: pgmap v284: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-09-20T18:44:18.475 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:18 smithi012 ceph-mon[65348]: pgmap v284: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.2 KiB/s wr, 7 op/s 2024-09-20T18:44:19.559 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: Detected new or changed devices on smithi174 2024-09-20T18:44:19.559 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all mds 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi012.jjsnmc"} : dispatch 2024-09-20T18:44:19.560 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi012.jjsnmc"}]': finished 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.zwzgjx"} : dispatch 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.zwzgjx"}]': finished 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all rgw 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all rbd-mirror 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all ceph-exporter 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all iscsi 2024-09-20T18:44:19.561 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:19 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.772 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: Detected new or changed devices on smithi174 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.773 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all mds 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi012.jjsnmc"} : dispatch 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi012.jjsnmc"}]': finished 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.zwzgjx"} : dispatch 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi174.zwzgjx"}]': finished 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all rgw 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all rbd-mirror 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.774 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all ceph-exporter 2024-09-20T18:44:19.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:19.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:19.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all iscsi 2024-09-20T18:44:19.775 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:19 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:21.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: pgmap v285: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-20T18:44:21.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:21.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T18:44:21.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T18:44:21.135 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:44:21.135 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:21.135 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T18:44:21.135 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:44:21.135 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T18:44:21.135 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:20 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:21.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: pgmap v285: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-20T18:44:21.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi012.nochla-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T18:44:21.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:20 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:22.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:21 smithi012 ceph-mon[65348]: Upgrade: Updating nfs.foo.0.0.smithi012.nochla (1/2) 2024-09-20T18:44:22.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:21 smithi012 ceph-mon[65348]: Creating key for client.nfs.foo.0.0.smithi012.nochla 2024-09-20T18:44:22.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:21 smithi012 ceph-mon[65348]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T18:44:22.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:21 smithi012 ceph-mon[65348]: Rados config object exists: conf-nfs.foo 2024-09-20T18:44:22.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:21 smithi012 ceph-mon[65348]: Creating key for client.nfs.foo.0.0.smithi012.nochla-rgw 2024-09-20T18:44:22.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:21 smithi012 ceph-mon[65348]: Bind address in nfs.foo.0.0.smithi012.nochla's ganesha conf is defaulting to empty 2024-09-20T18:44:22.051 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:21 smithi012 ceph-mon[65348]: Deploying daemon nfs.foo.0.0.smithi012.nochla on smithi012 2024-09-20T18:44:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:21 smithi174 ceph-mon[49279]: Upgrade: Updating nfs.foo.0.0.smithi012.nochla (1/2) 2024-09-20T18:44:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:21 smithi174 ceph-mon[49279]: Creating key for client.nfs.foo.0.0.smithi012.nochla 2024-09-20T18:44:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:21 smithi174 ceph-mon[49279]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-20T18:44:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:21 smithi174 ceph-mon[49279]: Rados config object exists: conf-nfs.foo 2024-09-20T18:44:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:21 smithi174 ceph-mon[49279]: Creating key for client.nfs.foo.0.0.smithi012.nochla-rgw 2024-09-20T18:44:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:21 smithi174 ceph-mon[49279]: Bind address in nfs.foo.0.0.smithi012.nochla's ganesha conf is defaulting to empty 2024-09-20T18:44:22.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:21 smithi174 ceph-mon[49279]: Deploying daemon nfs.foo.0.0.smithi012.nochla on smithi012 2024-09-20T18:44:23.167 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:22 smithi012 ceph-mon[65348]: pgmap v286: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-20T18:44:23.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:22 smithi174 ceph-mon[49279]: pgmap v286: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 7.1 KiB/s wr, 3 op/s 2024-09-20T18:44:24.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:23 smithi012 ceph-mon[65348]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.12:0/862280390"} : dispatch 2024-09-20T18:44:24.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:23 smithi174 ceph-mon[49279]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.12:0/862280390"} : dispatch 2024-09-20T18:44:24.907 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: pgmap v287: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 7.6 KiB/s wr, 4 op/s 2024-09-20T18:44:24.907 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: Evicting (and blocklisting) client session 24329 (172.21.15.12:0/862280390) 2024-09-20T18:44:24.907 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.12:0/862280390"}]': finished 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: pgmap v287: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 7.6 KiB/s wr, 4 op/s 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: Evicting (and blocklisting) client session 24329 (172.21.15.12:0/862280390) 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.12:0/862280390"}]': finished 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: osdmap e141: 8 total, 8 up, 8 in 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: osdmap e141: 8 total, 8 up, 8 in 2024-09-20T18:44:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:26.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: pgmap v289: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-20T18:44:26.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: Upgrade: Updating nfs.foo.1.0.smithi174.yhunhj (2/2) 2024-09-20T18:44:26.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: Rados config object exists: conf-nfs.foo 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj-rgw 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: Bind address in nfs.foo.1.0.smithi174.yhunhj's ganesha conf is defaulting to empty 2024-09-20T18:44:26.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:26.523 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:26 smithi174 ceph-mon[49279]: Deploying daemon nfs.foo.1.0.smithi174.yhunhj on smithi174 2024-09-20T18:44:26.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: pgmap v289: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 KiB/s rd, 614 B/s wr, 1 op/s 2024-09-20T18:44:26.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: Upgrade: Updating nfs.foo.1.0.smithi174.yhunhj (2/2) 2024-09-20T18:44:26.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:26.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: Rados config object exists: conf-nfs.foo 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: Creating key for client.nfs.foo.1.0.smithi174.yhunhj-rgw 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi174.yhunhj-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: Bind address in nfs.foo.1.0.smithi174.yhunhj's ganesha conf is defaulting to empty 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:26.727 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:26 smithi012 ceph-mon[65348]: Deploying daemon nfs.foo.1.0.smithi174.yhunhj on smithi174 2024-09-20T18:44:28.652 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:28 smithi174 ceph-mon[49279]: pgmap v290: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-09-20T18:44:28.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:28 smithi012 ceph-mon[65348]: pgmap v290: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 716 B/s wr, 2 op/s 2024-09-20T18:44:30.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: pgmap v291: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-09-20T18:44:30.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:30.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:30 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: pgmap v291: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.5 KiB/s wr, 5 op/s 2024-09-20T18:44:30.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:30.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:30.477 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:30 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:32.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:32 smithi012 ceph-mon[65348]: pgmap v292: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-20T18:44:32.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:32 smithi012 ceph-mon[65348]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/397365867"} : dispatch 2024-09-20T18:44:32.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:32.476 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:32 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:32.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:32 smithi174 ceph-mon[49279]: pgmap v292: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 1.6 KiB/s wr, 5 op/s 2024-09-20T18:44:32.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:32 smithi174 ceph-mon[49279]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/397365867"} : dispatch 2024-09-20T18:44:32.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:32.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:32 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:33.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:33 smithi174 ceph-mon[49279]: Evicting (and blocklisting) client session 24345 (172.21.15.174:0/397365867) 2024-09-20T18:44:33.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:33 smithi174 ceph-mon[49279]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/397365867"}]': finished 2024-09-20T18:44:33.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:33 smithi174 ceph-mon[49279]: osdmap e142: 8 total, 8 up, 8 in 2024-09-20T18:44:33.521 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:33 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:33.522 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:33 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:33.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:33 smithi012 ceph-mon[65348]: Evicting (and blocklisting) client session 24345 (172.21.15.174:0/397365867) 2024-09-20T18:44:33.629 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:33 smithi012 ceph-mon[65348]: from='mds.? [v2:172.21.15.12:6834/907145829,v1:172.21.15.12:6835/907145829]' entity='mds.foofs.smithi012.jjsnmc' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.174:0/397365867"}]': finished 2024-09-20T18:44:33.630 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:33 smithi012 ceph-mon[65348]: osdmap e142: 8 total, 8 up, 8 in 2024-09-20T18:44:33.630 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:33 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:33.630 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:33 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:34.170 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:44:34.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:34 smithi174 ceph-mon[49279]: pgmap v294: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-20T18:44:34.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:34.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:34 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (9m) 0s ago 24m 18.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (6m) 0s ago 23m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (6m) 2s ago 23m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (9m) 0s ago 23m 52.7M - dad864ee21e9 04fa6dbb5daa 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (46s) 0s ago 17m 37.8M - 19.3.0-5099-g88deadb8 441f9a3fe515 87f9941648bc 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (23s) 2s ago 17m 13.2M - 19.3.0-5099-g88deadb8 441f9a3fe515 c65aa297cba2 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (8m) 0s ago 25m 528M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (7m) 2s ago 23m 458M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:44:34.920 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (7m) 0s ago 25m 61.9M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (7m) 2s ago 23m 50.4M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (11s) 0s ago 17m 66.5M - 5.9 441f9a3fe515 b6c514e4fb25 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (6s) 2s ago 17m 15.7M - 5.9 441f9a3fe515 0b4874fa71bc 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (9m) 0s ago 23m 12.1M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (8m) 2s ago 23m 11.4M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (5m) 0s ago 22m 288M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (4m) 0s ago 21m 245M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (4m) 0s ago 21m 241M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (3m) 0s ago 21m 308M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (2m) 2s ago 20m 221M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (2m) 2s ago 20m 273M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b38346191a9d 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (102s) 2s ago 19m 214M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b5b9f7c2d041 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (66s) 2s ago 19m 259M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 cc57872beb57 2024-09-20T18:44:34.921 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (7m) 0s ago 22m 57.7M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:44:34.922 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:34 smithi012 ceph-mon[65348]: pgmap v294: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-20T18:44:34.922 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:34.922 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:34 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:35.379 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:44:35.379 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 8 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:44:35.380 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 14 2024-09-20T18:44:35.381 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:44:35.381 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": true, 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [ 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout: "mon", 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout: "osd", 2024-09-20T18:44:35.757 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr", 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout: "mds", 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout: "nfs", 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout: "crash" 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": "18/23 daemons upgraded", 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "Currently upgrading nfs daemons", 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:44:35.758 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:44:36.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:36.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:36.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:36.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2906403401' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:36.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:35 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2906403401' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.151 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:35 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:36.196 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:44:36.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:36 smithi012 ceph-mon[65348]: pgmap v295: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-20T18:44:36.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:36 smithi012 ceph-mon[65348]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:36.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:36 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/3820894611' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:44:37.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:36 smithi174 ceph-mon[49279]: pgmap v295: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-20T18:44:37.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:36 smithi174 ceph-mon[49279]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:44:37.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:36 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/3820894611' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:44:39.004 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: pgmap v296: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-20T18:44:39.004 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.004 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.005 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi012.nochla"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi012.nochla"}]': finished 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.yhunhj"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.yhunhj"}]': finished 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.006 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:38 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: pgmap v296: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi012.nochla"} : dispatch 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi012.nochla"}]': finished 2024-09-20T18:44:39.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.yhunhj"} : dispatch 2024-09-20T18:44:39.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi174.yhunhj"}]': finished 2024-09-20T18:44:39.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:39.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:39.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:38 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:44:40.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:39 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all nfs 2024-09-20T18:44:40.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:39 smithi174 ceph-mon[49279]: Upgrade: Setting container_image for all nvmeof 2024-09-20T18:44:40.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:39 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:40.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:39 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all nfs 2024-09-20T18:44:40.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:39 smithi012 ceph-mon[65348]: Upgrade: Setting container_image for all nvmeof 2024-09-20T18:44:40.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:39 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:40 smithi174 ceph-mon[49279]: pgmap v297: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-20T18:44:41.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:40 smithi174 ceph-mon[49279]: Upgrade: Updating grafana.smithi012 2024-09-20T18:44:41.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:40 smithi012 ceph-mon[65348]: pgmap v297: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-20T18:44:41.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:40 smithi012 ceph-mon[65348]: Upgrade: Updating grafana.smithi012 2024-09-20T18:44:42.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:41 smithi174 ceph-mon[49279]: Deploying daemon grafana.smithi012 on smithi012 2024-09-20T18:44:42.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:41 smithi012 ceph-mon[65348]: Deploying daemon grafana.smithi012 on smithi012 2024-09-20T18:44:43.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:42 smithi174 ceph-mon[49279]: pgmap v298: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-20T18:44:43.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:42 smithi012 ceph-mon[65348]: pgmap v298: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 3 op/s 2024-09-20T18:44:45.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:44 smithi174 ceph-mon[49279]: pgmap v299: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 KiB/s rd, 953 B/s wr, 3 op/s 2024-09-20T18:44:45.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:44 smithi012 ceph-mon[65348]: pgmap v299: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 KiB/s rd, 953 B/s wr, 3 op/s 2024-09-20T18:44:47.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:46 smithi174 ceph-mon[49279]: pgmap v300: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-20T18:44:47.225 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:46 smithi012 ceph-mon[65348]: pgmap v300: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-20T18:44:48.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:48 smithi012 ceph-mon[65348]: pgmap v301: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-20T18:44:49.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:48 smithi174 ceph-mon[49279]: pgmap v301: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-20T18:44:50.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:50 smithi012 ceph-mon[65348]: pgmap v302: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:51.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:50 smithi174 ceph-mon[49279]: pgmap v302: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:53.018 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:52 smithi012 ceph-mon[65348]: pgmap v303: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:53.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:52 smithi174 ceph-mon[49279]: pgmap v303: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:53.798 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:53.798 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:53.798 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:53 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:54.073 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:54.073 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:54.073 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:53 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:44:55.131 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:54 smithi012 ceph-mon[65348]: pgmap v304: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:44:55.131 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:54 smithi174 ceph-mon[49279]: pgmap v304: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:44:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:44:57.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:56 smithi012 ceph-mon[65348]: pgmap v305: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:57.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:56 smithi174 ceph-mon[49279]: pgmap v305: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:58.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:58 smithi012 ceph-mon[65348]: pgmap v306: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:58.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:58.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:58.977 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:58.977 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:44:58 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:59.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:58 smithi174 ceph-mon[49279]: pgmap v306: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:44:59.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:59.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:59.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:44:59.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:44:58 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:00.861 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:00 smithi012 ceph-mon[65348]: pgmap v307: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:01.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:00 smithi174 ceph-mon[49279]: pgmap v307: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:02.878 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: pgmap v308: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:02.878 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.878 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.878 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:45:02.878 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:45:02.878 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.879 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.879 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-20T18:45:02.879 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:45:02.879 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:45:02.879 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.879 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.879 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.880 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.881 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-20T18:45:02.882 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.883 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:02.884 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:02 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: pgmap v308: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-20T18:45:03.022 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.023 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-20T18:45:03.024 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-20T18:45:03.025 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:45:03.026 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.027 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:02 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:03.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:03 smithi012 ceph-mon[65348]: Upgrade: Finalizing container_image settings 2024-09-20T18:45:03.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:03 smithi012 ceph-mon[65348]: Upgrade: Complete! 2024-09-20T18:45:04.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:03 smithi174 ceph-mon[49279]: Upgrade: Finalizing container_image settings 2024-09-20T18:45:04.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:03 smithi174 ceph-mon[49279]: Upgrade: Complete! 2024-09-20T18:45:04.771 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:04 smithi174 ceph-mon[49279]: pgmap v309: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:04.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:04 smithi012 ceph-mon[65348]: pgmap v309: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:06.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:05 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:06.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:05 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:45:06.909 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:06 smithi012 ceph-mon[65348]: pgmap v310: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:07.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:06 smithi174 ceph-mon[49279]: pgmap v310: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:07.223 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph orch ps' 2024-09-20T18:45:07.862 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:07 smithi012 ceph-mon[65348]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:45:08.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:07 smithi174 ceph-mon[49279]: from='client.34528 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:45:08.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:08 smithi012 ceph-mon[65348]: pgmap v311: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:09.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:08 smithi174 ceph-mon[49279]: pgmap v311: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:09.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:10.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:10.877 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:10 smithi012 ceph-mon[65348]: pgmap v312: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:11.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:10 smithi174 ceph-mon[49279]: pgmap v312: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (10m) 14s ago 24m 18.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (6m) 14s ago 24m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (6m) 39s ago 24m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (19s) 14s ago 24m 40.8M - 10.4.8 5aad1d7cf004 8fffc3a92520 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (83s) 14s ago 18m 37.8M - 19.3.0-5099-g88deadb8 441f9a3fe515 87f9941648bc 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (60s) 39s ago 18m 13.2M - 19.3.0-5099-g88deadb8 441f9a3fe515 c65aa297cba2 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (9m) 14s ago 25m 536M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (8m) 39s ago 24m 458M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (8m) 14s ago 26m 61.9M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (7m) 39s ago 23m 50.4M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (48s) 14s ago 18m 68.5M - 5.9 441f9a3fe515 b6c514e4fb25 2024-09-20T18:45:12.107 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (43s) 39s ago 18m 15.7M - 5.9 441f9a3fe515 0b4874fa71bc 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (9m) 14s ago 24m 11.6M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (9m) 39s ago 23m 11.4M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (6m) 14s ago 23m 289M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (5m) 14s ago 22m 246M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (4m) 14s ago 22m 241M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (4m) 14s ago 21m 308M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (3m) 39s ago 21m 221M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (2m) 39s ago 20m 273M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b38346191a9d 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (2m) 39s ago 20m 214M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b5b9f7c2d041 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (103s) 39s ago 20m 259M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 cc57872beb57 2024-09-20T18:45:12.108 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (8m) 14s ago 23m 58.1M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:45:12.655 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph versions' 2024-09-20T18:45:12.968 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:12 smithi012 ceph-mon[65348]: pgmap v313: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:12.968 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:12 smithi012 ceph-mon[65348]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:45:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:12 smithi174 ceph-mon[49279]: pgmap v313: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:13.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:12 smithi174 ceph-mon[49279]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:45:14.846 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 8 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 14 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:45:14.847 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:45:15.021 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:14 smithi174 ceph-mon[49279]: pgmap v314: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:15.209 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:14 smithi012 ceph-mon[65348]: pgmap v314: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:15.640 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-20T18:45:15.955 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:15 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2784621365' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:16.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:15 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2784621365' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:45:16.950 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:16 smithi012 ceph-mon[65348]: pgmap v315: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:17.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:16 smithi174 ceph-mon[49279]: pgmap v315: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:17.479 INFO:teuthology.orchestra.run.smithi012.stdout:wait for servicemap items w/ changing names to refresh 2024-09-20T18:45:18.164 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'sleep 60' 2024-09-20T18:45:19.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:18 smithi012 ceph-mon[65348]: pgmap v316: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:19.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:18 smithi174 ceph-mon[49279]: pgmap v316: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:21.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:20 smithi012 ceph-mon[65348]: pgmap v317: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:21.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:20 smithi174 ceph-mon[49279]: pgmap v317: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:23.073 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:22 smithi174 ceph-mon[49279]: pgmap v318: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:23.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:22 smithi012 ceph-mon[65348]: pgmap v318: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:24 smithi012 ceph-mon[65348]: pgmap v319: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:25.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:24 smithi174 ceph-mon[49279]: pgmap v319: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:27.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:26 smithi012 ceph-mon[65348]: pgmap v320: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:26 smithi174 ceph-mon[49279]: pgmap v320: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:29.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:28 smithi012 ceph-mon[65348]: pgmap v321: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:28 smithi174 ceph-mon[49279]: pgmap v321: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:31.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:30 smithi012 ceph-mon[65348]: pgmap v322: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:31.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:30 smithi174 ceph-mon[49279]: pgmap v322: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:33.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:32 smithi012 ceph-mon[65348]: pgmap v323: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:32 smithi174 ceph-mon[49279]: pgmap v323: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:35.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:34 smithi012 ceph-mon[65348]: pgmap v324: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:35.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:34 smithi174 ceph-mon[49279]: pgmap v324: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:37.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:36 smithi012 ceph-mon[65348]: pgmap v325: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:37.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:36 smithi174 ceph-mon[49279]: pgmap v325: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:39.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:38 smithi012 ceph-mon[65348]: pgmap v326: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:39.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:38 smithi174 ceph-mon[49279]: pgmap v326: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:40.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:39 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:40.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:39 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:41.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:40 smithi012 ceph-mon[65348]: pgmap v327: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:41.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:40 smithi174 ceph-mon[49279]: pgmap v327: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:43.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:42 smithi012 ceph-mon[65348]: pgmap v328: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:43.291 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:42 smithi174 ceph-mon[49279]: pgmap v328: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:45.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:44 smithi012 ceph-mon[65348]: pgmap v329: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:44 smithi174 ceph-mon[49279]: pgmap v329: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:47.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:46 smithi012 ceph-mon[65348]: pgmap v330: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:47.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:46 smithi174 ceph-mon[49279]: pgmap v330: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:49.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:48 smithi012 ceph-mon[65348]: pgmap v331: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:49.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:48 smithi174 ceph-mon[49279]: pgmap v331: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:51.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:50 smithi012 ceph-mon[65348]: pgmap v332: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:51.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:50 smithi174 ceph-mon[49279]: pgmap v332: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:53.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:52 smithi012 ceph-mon[65348]: pgmap v333: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:52 smithi174 ceph-mon[49279]: pgmap v333: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:54 smithi012 ceph-mon[65348]: pgmap v334: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:55.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:54 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:55.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:54 smithi174 ceph-mon[49279]: pgmap v334: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:45:55.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:54 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:45:57.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:56 smithi012 ceph-mon[65348]: pgmap v335: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:57.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:56 smithi174 ceph-mon[49279]: pgmap v335: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:59.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:45:58 smithi012 ceph-mon[65348]: pgmap v336: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:45:59.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:45:58 smithi174 ceph-mon[49279]: pgmap v336: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:01.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:00 smithi012 ceph-mon[65348]: pgmap v337: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:01.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:00 smithi174 ceph-mon[49279]: pgmap v337: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:03.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:02 smithi012 ceph-mon[65348]: pgmap v338: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:03.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:02 smithi174 ceph-mon[49279]: pgmap v338: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:46:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:46:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:46:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:46:04.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:03 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:46:04.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-20T18:46:04.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-20T18:46:04.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-20T18:46:04.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:46:04.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:03 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' 2024-09-20T18:46:05.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:04 smithi012 ceph-mon[65348]: pgmap v339: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:05.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:04 smithi174 ceph-mon[49279]: pgmap v339: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:07.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:06 smithi012 ceph-mon[65348]: pgmap v340: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:07.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:06 smithi174 ceph-mon[49279]: pgmap v340: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:09.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:08 smithi012 ceph-mon[65348]: pgmap v341: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:09.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:08 smithi174 ceph-mon[49279]: pgmap v341: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:10.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:09 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:46:10.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:09 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:46:11.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:10 smithi012 ceph-mon[65348]: pgmap v342: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:11.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:10 smithi174 ceph-mon[49279]: pgmap v342: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:13.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:12 smithi012 ceph-mon[65348]: pgmap v343: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:13.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:12 smithi174 ceph-mon[49279]: pgmap v343: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:15.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:14 smithi012 ceph-mon[65348]: pgmap v344: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:15.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:14 smithi174 ceph-mon[49279]: pgmap v344: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:17.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:16 smithi012 ceph-mon[65348]: pgmap v345: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:17.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:16 smithi174 ceph-mon[49279]: pgmap v345: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:19.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:18 smithi012 ceph-mon[65348]: pgmap v346: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:19.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:18 smithi174 ceph-mon[49279]: pgmap v346: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:20.479 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph orch ps' 2024-09-20T18:46:21.060 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:20 smithi012 ceph-mon[65348]: pgmap v347: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:21.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:20 smithi174 ceph-mon[49279]: pgmap v347: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:alertmanager.smithi012 smithi012 *:9093,9094 running (11m) 84s ago 25m 18.5M - 0.27.0 11f11916f8cd 381860988e12 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi012 smithi012 running (8m) 84s ago 25m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 f1ef20cd7f0e 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:crash.smithi174 smithi174 running (8m) 110s ago 25m 6899k - 19.3.0-5099-g88deadb8 441f9a3fe515 ff6fb4c34736 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:grafana.smithi012 smithi012 *:3000 running (90s) 84s ago 25m 40.8M - 10.4.8 5aad1d7cf004 8fffc3a92520 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi012.jjsnmc smithi012 running (2m) 84s ago 19m 37.8M - 19.3.0-5099-g88deadb8 441f9a3fe515 87f9941648bc 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:mds.foofs.smithi174.zwzgjx smithi174 running (2m) 110s ago 19m 13.2M - 19.3.0-5099-g88deadb8 441f9a3fe515 c65aa297cba2 2024-09-20T18:46:22.523 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi012.xkxxrd smithi012 *:8443,9283,8765 running (10m) 84s ago 27m 536M - 19.3.0-5099-g88deadb8 441f9a3fe515 cbe827340c6d 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:mgr.smithi174.nxkptu smithi174 *:8443,9283,8765 running (9m) 110s ago 25m 458M - 19.3.0-5099-g88deadb8 441f9a3fe515 f9b9e8d19b98 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi012 smithi012 running (9m) 84s ago 27m 61.9M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 208d32fa88d1 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:mon.smithi174 smithi174 running (8m) 110s ago 25m 50.4M 2048M 19.3.0-5099-g88deadb8 441f9a3fe515 78d00d99c236 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.0.0.smithi012.nochla smithi012 *:2049 running (119s) 84s ago 19m 68.5M - 5.9 441f9a3fe515 b6c514e4fb25 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:nfs.foo.1.0.smithi174.yhunhj smithi174 *:2049 running (114s) 110s ago 19m 15.7M - 5.9 441f9a3fe515 0b4874fa71bc 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi012 smithi012 *:9100 running (11m) 84s ago 25m 11.6M - 1.7.0 72c9c2088986 26fd7930cd41 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:node-exporter.smithi174 smithi174 *:9100 running (10m) 110s ago 24m 11.4M - 1.7.0 72c9c2088986 8f6c2ad6b68c 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:osd.0 smithi012 running (7m) 84s ago 24m 289M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 280ecc270582 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:osd.1 smithi012 running (6m) 84s ago 23m 246M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 3fdcc6864263 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:osd.2 smithi012 running (5m) 84s ago 23m 241M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 7a10de3220f0 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:osd.3 smithi012 running (5m) 84s ago 22m 308M 1937M 19.3.0-5099-g88deadb8 441f9a3fe515 8b4734cb5150 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:osd.4 smithi174 running (4m) 110s ago 22m 221M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 532d357d165f 2024-09-20T18:46:22.524 INFO:teuthology.orchestra.run.smithi012.stdout:osd.5 smithi174 running (4m) 110s ago 22m 273M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b38346191a9d 2024-09-20T18:46:22.525 INFO:teuthology.orchestra.run.smithi012.stdout:osd.6 smithi174 running (3m) 110s ago 21m 214M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 b5b9f7c2d041 2024-09-20T18:46:22.525 INFO:teuthology.orchestra.run.smithi012.stdout:osd.7 smithi174 running (2m) 110s ago 21m 259M 2704M 19.3.0-5099-g88deadb8 441f9a3fe515 cc57872beb57 2024-09-20T18:46:22.525 INFO:teuthology.orchestra.run.smithi012.stdout:prometheus.smithi012 smithi012 *:9095 running (9m) 84s ago 24m 58.1M - 2.51.0 1d3b7f56885b bc3a44c185ca 2024-09-20T18:46:23.145 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:22 smithi012 ceph-mon[65348]: pgmap v348: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:23.174 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph orch upgrade status' 2024-09-20T18:46:23.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:22 smithi174 ceph-mon[49279]: pgmap v348: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:23.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:23 smithi012 ceph-mon[65348]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:24.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:23 smithi174 ceph-mon[49279]: from='client.34540 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:25.065 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:24 smithi012 ceph-mon[65348]: pgmap v349: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:25.066 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:24 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:46:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:24 smithi174 ceph-mon[49279]: pgmap v349: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:25.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:24 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:46:25.427 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:46:25.427 INFO:teuthology.orchestra.run.smithi012.stdout: "target_image": null, 2024-09-20T18:46:25.427 INFO:teuthology.orchestra.run.smithi012.stdout: "in_progress": false, 2024-09-20T18:46:25.428 INFO:teuthology.orchestra.run.smithi012.stdout: "which": "", 2024-09-20T18:46:25.428 INFO:teuthology.orchestra.run.smithi012.stdout: "services_complete": [], 2024-09-20T18:46:25.428 INFO:teuthology.orchestra.run.smithi012.stdout: "progress": null, 2024-09-20T18:46:25.428 INFO:teuthology.orchestra.run.smithi012.stdout: "message": "", 2024-09-20T18:46:25.428 INFO:teuthology.orchestra.run.smithi012.stdout: "is_paused": false 2024-09-20T18:46:25.428 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:46:26.120 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph health detail' 2024-09-20T18:46:27.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:26 smithi012 ceph-mon[65348]: pgmap v350: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:27.134 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:26 smithi012 ceph-mon[65348]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:26 smithi174 ceph-mon[49279]: pgmap v350: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:27.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:26 smithi174 ceph-mon[49279]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:28.267 INFO:teuthology.orchestra.run.smithi012.stdout:HEALTH_OK 2024-09-20T18:46:28.903 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph versions' 2024-09-20T18:46:29.213 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:28 smithi012 ceph-mon[65348]: pgmap v351: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:29.213 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:28 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/461187864' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:46:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:28 smithi174 ceph-mon[49279]: pgmap v351: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:29.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:28 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/461187864' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-20T18:46:31.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:30 smithi012 ceph-mon[65348]: pgmap v352: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout:{ 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: "mon": { 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: "mgr": { 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: "osd": { 2024-09-20T18:46:31.258 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 8 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout: "mds": { 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 2 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout: "overall": { 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)": 14 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:46:31.259 INFO:teuthology.orchestra.run.smithi012.stdout:} 2024-09-20T18:46:31.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:30 smithi174 ceph-mon[49279]: pgmap v352: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:31.875 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-20T18:46:32.188 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:31 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2062214570' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:46:32.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:31 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2062214570' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:46:33.095 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:32 smithi012 ceph-mon[65348]: pgmap v353: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:33.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:32 smithi174 ceph-mon[49279]: pgmap v353: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:33.920 INFO:teuthology.orchestra.run.smithi012.stdout:true 2024-09-20T18:46:34.211 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:33 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2528588359' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:46:34.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:33 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2528588359' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:46:34.528 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -e sha1=88deadb8082f4a33832e228ccefc59cedba158a5 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-20T18:46:35.181 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:34 smithi012 ceph-mon[65348]: pgmap v354: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:35.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:34 smithi174 ceph-mon[49279]: pgmap v354: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:36.934 INFO:teuthology.orchestra.run.smithi012.stdout: "ceph version 19.3.0-5099-g88deadb8 (88deadb8082f4a33832e228ccefc59cedba158a5) squid (dev)" 2024-09-20T18:46:37.217 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:36 smithi012 ceph-mon[65348]: pgmap v355: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:37.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:36 smithi174 ceph-mon[49279]: pgmap v355: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:37.509 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-20T18:46:37.510 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-20T18:46:37.510 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- ceph orch ls -f json 2024-09-20T18:46:38.154 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:37 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/2559057819' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:46:38.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:37 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/2559057819' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-20T18:46:39.042 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:38 smithi012 ceph-mon[65348]: pgmap v356: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:39.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:38 smithi174 ceph-mon[49279]: pgmap v356: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:39.637 INFO:teuthology.orchestra.run.smithi012.stdout: 2024-09-20T18:46:39.637 INFO:teuthology.orchestra.run.smithi012.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-20T18:19:56.492926Z", "last_refresh": "2024-09-20T18:44:57.686460Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-20T18:19:49.875361Z", "last_refresh": "2024-09-20T18:44:32.010667Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-20T18:19:52.595112Z", "last_refresh": "2024-09-20T18:44:57.686629Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-20T18:26:31.056849Z", "last_refresh": "2024-09-20T18:44:32.011288Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-20T18:19:48.070520Z", "last_refresh": "2024-09-20T18:44:32.010765Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi012:172.21.15.12=smithi012", "smithi174:172.21.15.174=smithi174"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-20T18:20:41.415183Z", "last_refresh": "2024-09-20T18:44:32.010839Z", "running": 2, "size": 2}}, {"events": ["2024-09-20T18:46:03.394049Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-20T18:26:52.917785Z", "last_refresh": "2024-09-20T18:44:32.011360Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-20T18:19:54.379599Z", "last_refresh": "2024-09-20T18:44:32.010911Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2", "last_refresh": "2024-09-20T18:44:32.011000Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-20T18:19:51.374551Z", "last_refresh": "2024-09-20T18:44:57.686777Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-20T18:46:40.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:39 smithi012 ceph-mon[65348]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:46:40.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:39 smithi174 ceph-mon[49279]: from='mgr.34104 172.21.15.12:0/1311498423' entity='mgr.smithi012.xkxxrd' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-20T18:46:40.317 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-20T18:46:40.317 DEBUG:teuthology.parallel:result is None 2024-09-20T18:46:40.317 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-20T18:46:40.327 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:46:40.327 DEBUG:teuthology.orchestra.run.smithi012:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-20T18:46:40.356 INFO:teuthology.orchestra.run.smithi012.stderr:+ umount /mnt/foo 2024-09-20T18:46:40.367 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T18:46:40.376 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:46:40.376 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-20T18:46:40.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:40 smithi012 ceph-mon[65348]: pgmap v357: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:40.976 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:40 smithi012 ceph-mon[65348]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:46:41.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:40 smithi174 ceph-mon[49279]: pgmap v357: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:41.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:40 smithi174 ceph-mon[49279]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-20T18:46:42.504 INFO:teuthology.orchestra.run.smithi012.stdout: "foo" 2024-09-20T18:46:43.121 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-20T18:46:43.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:42 smithi012 ceph-mon[65348]: pgmap v358: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:43.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:42 smithi174 ceph-mon[49279]: pgmap v358: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:44.119 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:43 smithi012 ceph-mon[65348]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:44.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:43 smithi174 ceph-mon[49279]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:45.202 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:44 smithi012 ceph-mon[65348]: pgmap v359: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:45.226 INFO:teuthology.orchestra.run.smithi012.stdout:[ 2024-09-20T18:46:45.226 INFO:teuthology.orchestra.run.smithi012.stdout: { 2024-09-20T18:46:45.226 INFO:teuthology.orchestra.run.smithi012.stdout: "access_type": "RW", 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "clients": [], 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "cluster_id": "foo", 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "export_id": 1, 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "fsal": { 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "fs_name": "foofs", 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "name": "CEPH", 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "user_id": "nfs.foo.1" 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: }, 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "path": "/", 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "protocols": [ 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: 4 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: ], 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "pseudo": "/fake", 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "security_label": true, 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "squash": "none", 2024-09-20T18:46:45.227 INFO:teuthology.orchestra.run.smithi012.stdout: "transports": [ 2024-09-20T18:46:45.228 INFO:teuthology.orchestra.run.smithi012.stdout: "TCP" 2024-09-20T18:46:45.228 INFO:teuthology.orchestra.run.smithi012.stdout: ] 2024-09-20T18:46:45.228 INFO:teuthology.orchestra.run.smithi012.stdout: } 2024-09-20T18:46:45.228 INFO:teuthology.orchestra.run.smithi012.stdout:] 2024-09-20T18:46:45.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:44 smithi174 ceph-mon[49279]: pgmap v359: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-20T18:46:45.869 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'rados -p .nfs --all ls -' 2024-09-20T18:46:47.197 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:46 smithi012 ceph-mon[65348]: pgmap v360: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:47.198 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:46 smithi012 ceph-mon[65348]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:47.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:46 smithi174 ceph-mon[49279]: pgmap v360: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-20T18:46:47.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:46 smithi174 ceph-mon[49279]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-20T18:46:47.627 INFO:teuthology.orchestra.run.smithi012.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-20T18:46:47.628 INFO:teuthology.orchestra.run.smithi012.stdout:foo grace 2024-09-20T18:46:47.628 INFO:teuthology.orchestra.run.smithi012.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-20T18:46:47.628 INFO:teuthology.orchestra.run.smithi012.stdout:foo conf-nfs.foo 2024-09-20T18:46:47.628 INFO:teuthology.orchestra.run.smithi012.stdout:foo export-1 2024-09-20T18:46:48.354 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-20T18:46:48.365 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:46:48.366 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid c01c9b48-777c-11ef-baf5-efdc52797490 -- bash -c 'set -ex 2024-09-20T18:46:48.366 DEBUG:teuthology.orchestra.run.smithi012:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-20T18:46:48.366 DEBUG:teuthology.orchestra.run.smithi012:> ' 2024-09-20T18:46:49.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:48 smithi012 ceph-mon[65348]: pgmap v361: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:46:49.226 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:48 smithi012 ceph-mon[65348]: mgrmap e44: smithi012.xkxxrd(active, since 9m), standbys: smithi174.nxkptu 2024-09-20T18:46:49.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:48 smithi174 ceph-mon[49279]: pgmap v361: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:46:49.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:48 smithi174 ceph-mon[49279]: mgrmap e44: smithi012.xkxxrd(active, since 9m), standbys: smithi174.nxkptu 2024-09-20T18:46:50.217 INFO:teuthology.orchestra.run.smithi012.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-20T18:46:50.492 INFO:teuthology.orchestra.run.smithi012.stderr:+ [[ 7 -gt 2 ]] 2024-09-20T18:46:51.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:50 smithi012 ceph-mon[65348]: pgmap v362: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:46:51.088 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:50 smithi012 ceph-mon[65348]: from='client.? 172.21.15.12:0/1925851317' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-20T18:46:51.114 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-20T18:46:51.123 INFO:tasks.cephadm:Teardown begin 2024-09-20T18:46:51.123 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T18:46:51.152 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T18:46:51.184 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-20T18:46:51.184 DEBUG:teuthology.orchestra.run.smithi012:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-20T18:46:51.208 DEBUG:teuthology.orchestra.run.smithi174:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-20T18:46:51.242 INFO:tasks.cephadm:Stopping all daemons... 2024-09-20T18:46:51.242 INFO:tasks.cephadm.mon.smithi012:Stopping mon.smithi012... 2024-09-20T18:46:51.242 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012 2024-09-20T18:46:51.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:50 smithi174 ceph-mon[49279]: pgmap v362: 97 pgs: 97 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-20T18:46:51.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:50 smithi174 ceph-mon[49279]: from='client.? 172.21.15.12:0/1925851317' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-20T18:46:51.428 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:51 smithi012 systemd[1]: Stopping Ceph mon.smithi012 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:46:51.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:51 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[65344]: 2024-09-20T18:46:51.642+0000 7ff07ad53640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi012 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:46:51.726 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:51 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012[65344]: 2024-09-20T18:46:51.642+0000 7ff07ad53640 -1 mon.smithi012@0(leader) e3 *** Got Signal Terminated *** 2024-09-20T18:46:52.035 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:51 smithi012 podman[95031]: 2024-09-20 18:46:51.754873825 +0000 UTC m=+0.324768400 container died 208d32fa88d140e55acff111bdee058c0bb0a3df5e0bf8070e358942ca5b20b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-20T18:46:52.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:52 smithi012 podman[95031]: 2024-09-20 18:46:52.134071858 +0000 UTC m=+0.703966422 container cleanup 208d32fa88d140e55acff111bdee058c0bb0a3df5e0bf8070e358942ca5b20b0 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:46:52.426 INFO:journalctl@ceph.mon.smithi012.smithi012.stdout:Sep 20 18:46:52 smithi012 bash[95031]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi012 2024-09-20T18:46:52.597 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi012.service' 2024-09-20T18:46:52.651 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:46:52.652 INFO:tasks.cephadm.mon.smithi012:Stopped mon.smithi012 2024-09-20T18:46:52.652 INFO:tasks.cephadm.mon.smithi174:Stopping mon.smithi174... 2024-09-20T18:46:52.652 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi174 2024-09-20T18:46:52.957 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:52 smithi174 systemd[1]: Stopping Ceph mon.smithi174 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:46:53.271 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:52 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174[49275]: 2024-09-20T18:46:52.940+0000 7f511ccdf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi174 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:46:53.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:52 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174[49275]: 2024-09-20T18:46:52.940+0000 7f511ccdf640 -1 mon.smithi174@1(peon) e3 *** Got Signal Terminated *** 2024-09-20T18:46:53.272 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:53 smithi174 podman[68056]: 2024-09-20 18:46:53.083397302 +0000 UTC m=+0.255183657 container died 78d00d99c236bd6f04a534ecfb989be3c5e1b28296c415444fd0750e2e4ea3b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS) 2024-09-20T18:46:53.842 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:53 smithi174 podman[68056]: 2024-09-20 18:46:53.557688531 +0000 UTC m=+0.729474878 container cleanup 78d00d99c236bd6f04a534ecfb989be3c5e1b28296c415444fd0750e2e4ea3b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, ceph=True) 2024-09-20T18:46:53.842 INFO:journalctl@ceph.mon.smithi174.smithi174.stdout:Sep 20 18:46:53 smithi174 bash[68056]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-mon-smithi174 2024-09-20T18:46:54.003 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@mon.smithi174.service' 2024-09-20T18:46:54.048 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:46:54.049 INFO:tasks.cephadm.mon.smithi174:Stopped mon.smithi174 2024-09-20T18:46:54.049 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-20T18:46:54.049 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.0 2024-09-20T18:46:54.371 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:46:54 smithi012 systemd[1]: Stopping Ceph osd.0 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:46:54.726 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:46:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[70161]: 2024-09-20T18:46:54.354+0000 7f6d15bbe640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:46:54.726 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:46:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[70161]: 2024-09-20T18:46:54.354+0000 7f6d15bbe640 -1 osd.0 142 *** Got signal Terminated *** 2024-09-20T18:46:54.726 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:46:54 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0[70161]: 2024-09-20T18:46:54.354+0000 7f6d15bbe640 -1 osd.0 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:00.188 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:46:59 smithi012 podman[95163]: 2024-09-20 18:46:59.873992014 +0000 UTC m=+5.640218606 container died 280ecc2705828fb8abc8af338c0eff056dbdeceb6862d6b9cbb99f39c3f8d8ad (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-20T18:47:00.476 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:00 smithi012 podman[95163]: 2024-09-20 18:47:00.254189971 +0000 UTC m=+6.020416555 container cleanup 280ecc2705828fb8abc8af338c0eff056dbdeceb6862d6b9cbb99f39c3f8d8ad (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-20T18:47:00.476 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:00 smithi012 bash[95163]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0 2024-09-20T18:47:00.902 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:00 smithi012 podman[95181]: 2024-09-20 18:47:00.547323771 +0000 UTC m=+0.671666760 container remove 280ecc2705828fb8abc8af338c0eff056dbdeceb6862d6b9cbb99f39c3f8d8ad (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-20T18:47:01.192 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:00 smithi012 podman[95269]: 2024-09-20 18:47:00.789297038 +0000 UTC m=+0.016753902 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:01.192 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:00 smithi012 podman[95269]: 2024-09-20 18:47:00.906440511 +0000 UTC m=+0.133897376 container create a5d5ef32f32f5e4a5dbae106af55fbf914386526c826b5d629110becb07e88d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-20T18:47:01.192 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:01 smithi012 podman[95269]: 2024-09-20 18:47:01.176214361 +0000 UTC m=+0.403671225 container init a5d5ef32f32f5e4a5dbae106af55fbf914386526c826b5d629110becb07e88d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-20T18:47:01.476 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:01 smithi012 podman[95269]: 2024-09-20 18:47:01.179801623 +0000 UTC m=+0.407258495 container start a5d5ef32f32f5e4a5dbae106af55fbf914386526c826b5d629110becb07e88d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-20T18:47:01.476 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:01 smithi012 podman[95269]: 2024-09-20 18:47:01.276050933 +0000 UTC m=+0.503507807 container attach a5d5ef32f32f5e4a5dbae106af55fbf914386526c826b5d629110becb07e88d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-20T18:47:01.477 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:01 smithi012 podman[95269]: 2024-09-20 18:47:01.380709917 +0000 UTC m=+0.608166783 container died a5d5ef32f32f5e4a5dbae106af55fbf914386526c826b5d629110becb07e88d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-20T18:47:01.975 INFO:journalctl@ceph.osd.0.smithi012.stdout:Sep 20 18:47:01 smithi012 podman[95309]: 2024-09-20 18:47:01.946397096 +0000 UTC m=+0.554623512 container remove a5d5ef32f32f5e4a5dbae106af55fbf914386526c826b5d629110becb07e88d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-20T18:47:01.982 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.0.service' 2024-09-20T18:47:02.027 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:02.027 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-20T18:47:02.027 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-20T18:47:02.027 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.1 2024-09-20T18:47:02.289 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:02 smithi012 systemd[1]: Stopping Ceph osd.1 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:47:02.726 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:02 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[72666]: 2024-09-20T18:47:02.382+0000 7fbb0b30e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:47:02.726 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:02 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[72666]: 2024-09-20T18:47:02.382+0000 7fbb0b30e640 -1 osd.1 142 *** Got signal Terminated *** 2024-09-20T18:47:02.726 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:02 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1[72666]: 2024-09-20T18:47:02.382+0000 7fbb0b30e640 -1 osd.1 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:08.227 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:07 smithi012 podman[95390]: 2024-09-20 18:47:07.940050924 +0000 UTC m=+5.691731648 container died 3fdcc68642636648c442c13c3dc05cd271c2abf54fd96813ed9c6c41e6f7e4ae (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-20T18:47:08.565 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:08 smithi012 podman[95390]: 2024-09-20 18:47:08.312314852 +0000 UTC m=+6.063995560 container cleanup 3fdcc68642636648c442c13c3dc05cd271c2abf54fd96813ed9c6c41e6f7e4ae (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-09-20T18:47:08.566 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:08 smithi012 bash[95390]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1 2024-09-20T18:47:08.914 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:08 smithi012 podman[95416]: 2024-09-20 18:47:08.550097354 +0000 UTC m=+0.611289468 container remove 3fdcc68642636648c442c13c3dc05cd271c2abf54fd96813ed9c6c41e6f7e4ae (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-20T18:47:09.186 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:08 smithi012 podman[95510]: 2024-09-20 18:47:08.80124987 +0000 UTC m=+0.016604551 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:09.187 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:08 smithi012 podman[95510]: 2024-09-20 18:47:08.967805779 +0000 UTC m=+0.183160460 container create 2f9c895e97821a8c80af25b80ade8b7997220c1eb3e0e7d4b5ed99a3a2887aea (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:47:09.476 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:09 smithi012 podman[95510]: 2024-09-20 18:47:09.170603414 +0000 UTC m=+0.385958108 container init 2f9c895e97821a8c80af25b80ade8b7997220c1eb3e0e7d4b5ed99a3a2887aea (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-20T18:47:09.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:09 smithi012 podman[95510]: 2024-09-20 18:47:09.174795977 +0000 UTC m=+0.390150661 container start 2f9c895e97821a8c80af25b80ade8b7997220c1eb3e0e7d4b5ed99a3a2887aea (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-20T18:47:09.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:09 smithi012 podman[95510]: 2024-09-20 18:47:09.270160257 +0000 UTC m=+0.485514952 container attach 2f9c895e97821a8c80af25b80ade8b7997220c1eb3e0e7d4b5ed99a3a2887aea (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:47:09.477 INFO:journalctl@ceph.osd.1.smithi012.stdout:Sep 20 18:47:09 smithi012 podman[95510]: 2024-09-20 18:47:09.372361822 +0000 UTC m=+0.587716506 container died 2f9c895e97821a8c80af25b80ade8b7997220c1eb3e0e7d4b5ed99a3a2887aea (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-20T18:47:10.034 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.1.service' 2024-09-20T18:47:10.077 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:10.078 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-20T18:47:10.078 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-20T18:47:10.078 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.2 2024-09-20T18:47:10.410 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:10 smithi012 systemd[1]: Stopping Ceph osd.2 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:47:10.726 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:10 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[75241]: 2024-09-20T18:47:10.393+0000 7f7ed6fe0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:47:10.726 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:10 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[75241]: 2024-09-20T18:47:10.393+0000 7f7ed6fe0640 -1 osd.2 142 *** Got signal Terminated *** 2024-09-20T18:47:10.726 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:10 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2[75241]: 2024-09-20T18:47:10.393+0000 7f7ed6fe0640 -1 osd.2 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:16.093 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:15 smithi012 podman[95613]: 2024-09-20 18:47:15.809706616 +0000 UTC m=+5.507829500 container died 7a10de3220f0d1e4b414ed577910162c73e38cabed1527fb666636eb078bac14 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-20T18:47:16.461 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:16 smithi012 podman[95613]: 2024-09-20 18:47:16.168023134 +0000 UTC m=+5.866146005 container cleanup 7a10de3220f0d1e4b414ed577910162c73e38cabed1527fb666636eb078bac14 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-20T18:47:16.461 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:16 smithi012 bash[95613]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2 2024-09-20T18:47:16.726 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:16 smithi012 podman[95630]: 2024-09-20 18:47:16.44417785 +0000 UTC m=+0.628637094 container remove 7a10de3220f0d1e4b414ed577910162c73e38cabed1527fb666636eb078bac14 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD) 2024-09-20T18:47:17.072 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:16 smithi012 podman[95720]: 2024-09-20 18:47:16.706699532 +0000 UTC m=+0.016509515 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:17.072 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:16 smithi012 podman[95720]: 2024-09-20 18:47:16.811938328 +0000 UTC m=+0.121748338 container create b60dd5a76ecc3753ed11da6b168bb14f80085090f7314cc7339e3b19c545910b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-20T18:47:17.476 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:17 smithi012 podman[95720]: 2024-09-20 18:47:17.056187313 +0000 UTC m=+0.365997293 container init b60dd5a76ecc3753ed11da6b168bb14f80085090f7314cc7339e3b19c545910b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2) 2024-09-20T18:47:17.476 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:17 smithi012 podman[95720]: 2024-09-20 18:47:17.060036722 +0000 UTC m=+0.369846695 container start b60dd5a76ecc3753ed11da6b168bb14f80085090f7314cc7339e3b19c545910b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD) 2024-09-20T18:47:17.476 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:17 smithi012 podman[95720]: 2024-09-20 18:47:17.139496728 +0000 UTC m=+0.449306704 container attach b60dd5a76ecc3753ed11da6b168bb14f80085090f7314cc7339e3b19c545910b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:47:17.477 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:17 smithi012 podman[95720]: 2024-09-20 18:47:17.253019579 +0000 UTC m=+0.562829560 container died b60dd5a76ecc3753ed11da6b168bb14f80085090f7314cc7339e3b19c545910b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-20T18:47:17.811 INFO:journalctl@ceph.osd.2.smithi012.stdout:Sep 20 18:47:17 smithi012 podman[95744]: 2024-09-20 18:47:17.784293774 +0000 UTC m=+0.521407395 container remove b60dd5a76ecc3753ed11da6b168bb14f80085090f7314cc7339e3b19c545910b (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-20T18:47:17.816 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.2.service' 2024-09-20T18:47:17.856 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:17.856 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-20T18:47:17.856 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-20T18:47:17.856 DEBUG:teuthology.orchestra.run.smithi012:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.3 2024-09-20T18:47:18.226 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:17 smithi012 systemd[1]: Stopping Ceph osd.3 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:47:18.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:18 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:47:18.212+0000 7f109bf1a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:47:18.726 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:18 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:47:18.212+0000 7f109bf1a640 -1 osd.3 142 *** Got signal Terminated *** 2024-09-20T18:47:18.727 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:18 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:47:18.212+0000 7f109bf1a640 -1 osd.3 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:20.772 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:20 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:20.351+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:21.210 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:20 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:20.834+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:21.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:21 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:21.193+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:21.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:21 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:21.374+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:21.809 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:21 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:21.533+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:22.241 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:21 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:21.792+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:22.476 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:22 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:47:22.003+0000 7f109852f640 -1 osd.3 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.559004+0000 front 2024-09-20T18:46:57.559156+0000 (oldest deadline 2024-09-20T18:47:21.659125+0000) 2024-09-20T18:47:22.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:22 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:22.225+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:22.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:22 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:22.421+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:22.832 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:22 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:22.514+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:23.233 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:22 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:22.816+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:23.476 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:22 smithi012 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3[77699]: 2024-09-20T18:47:22.981+0000 7f109852f640 -1 osd.3 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.559004+0000 front 2024-09-20T18:46:57.559156+0000 (oldest deadline 2024-09-20T18:47:21.659125+0000) 2024-09-20T18:47:23.521 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:23 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:23.454+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:23.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:23 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:23.217+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:23.913 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:23 smithi012 podman[95823]: 2024-09-20 18:47:23.611074805 +0000 UTC m=+5.543354670 container died 8b4734cb5150e05c432e1debddfaf8d4053dae2f70c07eefdd5cac76caa09893 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:47:24.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:23 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:23.777+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:24.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:23 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:23.533+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:24.226 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:23 smithi012 podman[95823]: 2024-09-20 18:47:23.994849073 +0000 UTC m=+5.927128936 container cleanup 8b4734cb5150e05c432e1debddfaf8d4053dae2f70c07eefdd5cac76caa09893 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-20T18:47:24.226 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:23 smithi012 bash[95823]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3 2024-09-20T18:47:24.502 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:24.207+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:24.617 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:24 smithi012 podman[95855]: 2024-09-20 18:47:24.26292509 +0000 UTC m=+0.650107468 container remove 8b4734cb5150e05c432e1debddfaf8d4053dae2f70c07eefdd5cac76caa09893 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-20T18:47:24.766 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:24.511+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:24.767 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:24.486+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:24.873 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:24 smithi012 podman[95942]: 2024-09-20 18:47:24.504591781 +0000 UTC m=+0.016834106 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:24.873 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:24 smithi012 podman[95942]: 2024-09-20 18:47:24.605218817 +0000 UTC m=+0.117461186 container create 21cd62db229346ea8e4c51e60ae54bcfd15dca7543428ec6955d099a75a5e889 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-20T18:47:24.873 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:24 smithi012 podman[95942]: 2024-09-20 18:47:24.799308742 +0000 UTC m=+0.311551059 container init 21cd62db229346ea8e4c51e60ae54bcfd15dca7543428ec6955d099a75a5e889 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-20T18:47:24.873 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:24 smithi012 podman[95942]: 2024-09-20 18:47:24.803266216 +0000 UTC m=+0.315508532 container start 21cd62db229346ea8e4c51e60ae54bcfd15dca7543428ec6955d099a75a5e889 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-20T18:47:24.873 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:24 smithi012 podman[95942]: 2024-09-20 18:47:24.857455327 +0000 UTC m=+0.369697668 container attach 21cd62db229346ea8e4c51e60ae54bcfd15dca7543428ec6955d099a75a5e889 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_CLEAN=True) 2024-09-20T18:47:25.022 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:24 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:24.750+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:25.226 INFO:journalctl@ceph.osd.3.smithi012.stdout:Sep 20 18:47:24 smithi012 podman[95942]: 2024-09-20 18:47:24.997354794 +0000 UTC m=+0.509597119 container died 21cd62db229346ea8e4c51e60ae54bcfd15dca7543428ec6955d099a75a5e889 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-20T18:47:25.521 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:25.165+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:25.628 DEBUG:teuthology.orchestra.run.smithi012:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.3.service' 2024-09-20T18:47:25.668 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:25.668 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-20T18:47:25.668 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-20T18:47:25.668 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.4 2024-09-20T18:47:25.984 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:25.532+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:25.985 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:25 smithi174 systemd[1]: Stopping Ceph osd.4 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:47:25.985 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:25.716+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:25.985 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:25.514+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:26.271 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:26 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:26.210+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:26.272 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:25.968+0000 7f0ffa318640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:47:26.272 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:25.968+0000 7f0ffa318640 -1 osd.4 142 *** Got signal Terminated *** 2024-09-20T18:47:26.272 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:25 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:25.968+0000 7f0ffa318640 -1 osd.4 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:26.763 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:26 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:26.487+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:26.763 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:26 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:26.472+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:27.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:26 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:26.746+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:27.491 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:27.193+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:27.765 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:27.482+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:27.766 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:27.475+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:28.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:27 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:27.749+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:28.505 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:28 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:28.183+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:28.771 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:28 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:28.500+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:28.772 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:28 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:28.718+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:28.772 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:28 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:28.718+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:06.272475+0000 front 2024-09-20T18:47:06.272358+0000 (oldest deadline 2024-09-20T18:47:27.972199+0000) 2024-09-20T18:47:28.772 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:28 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:28.488+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:28.772 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:28 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:28.488+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:29.471 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:29 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:29.153+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:29.772 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:29 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:29.546+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:29.772 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:29 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:29.697+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:29.772 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:29 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:29.697+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:06.272475+0000 front 2024-09-20T18:47:06.272358+0000 (oldest deadline 2024-09-20T18:47:27.972199+0000) 2024-09-20T18:47:29.772 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:29 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:29.455+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:29.772 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:29 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:29.455+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:30.466 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:30 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:30.181+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:30.736 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:30 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:30.512+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:30.737 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:30 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:30.450+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:30.737 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:30 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:30.450+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:31.021 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:30 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:30.720+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:53.971681+0000 front 2024-09-20T18:46:53.971640+0000 (oldest deadline 2024-09-20T18:47:19.871133+0000) 2024-09-20T18:47:31.022 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:30 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4[53233]: 2024-09-20T18:47:30.720+0000 7f0ff2925640 -1 osd.4 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:06.272475+0000 front 2024-09-20T18:47:06.272358+0000 (oldest deadline 2024-09-20T18:47:27.972199+0000) 2024-09-20T18:47:31.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:31 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:31.230+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:31.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:31 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:31.230+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:31.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:31 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:31.461+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:31.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:31 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:31.461+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:31.868 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:31 smithi174 podman[68223]: 2024-09-20 18:47:31.509591847 +0000 UTC m=+5.663621064 container died 532d357d165fd76d2a67faa3826c9b87f26fdbaa589b94506102e7d73ea99230 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-20T18:47:31.868 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:31 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:31.536+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:31.869 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:31 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:31.536+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:32.227 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68223]: 2024-09-20 18:47:32.0183721 +0000 UTC m=+6.172401348 container cleanup 532d357d165fd76d2a67faa3826c9b87f26fdbaa589b94506102e7d73ea99230 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:47:32.227 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 bash[68223]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4 2024-09-20T18:47:32.487 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68238]: 2024-09-20 18:47:32.211502954 +0000 UTC m=+0.698209260 container remove 532d357d165fd76d2a67faa3826c9b87f26fdbaa589b94506102e7d73ea99230 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-20T18:47:32.488 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:32.280+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:32.488 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:32.280+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:32.764 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68326]: 2024-09-20 18:47:32.529054835 +0000 UTC m=+0.106459803 container create 593ec7b95c610fa68cec97926bc501d0e248230b1cf57c781f30d27f225e91dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:47:32.764 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68326]: 2024-09-20 18:47:32.43862765 +0000 UTC m=+0.016032599 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:32.764 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68326]: 2024-09-20 18:47:32.748156589 +0000 UTC m=+0.325561521 container init 593ec7b95c610fa68cec97926bc501d0e248230b1cf57c781f30d27f225e91dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:47:32.765 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:32.471+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:32.765 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:32.471+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:32.765 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:32.578+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:32.765 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:32 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:32.578+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:33.022 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68326]: 2024-09-20 18:47:32.75227608 +0000 UTC m=+0.329681005 container start 593ec7b95c610fa68cec97926bc501d0e248230b1cf57c781f30d27f225e91dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-20T18:47:33.022 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68326]: 2024-09-20 18:47:32.839475227 +0000 UTC m=+0.416880160 container attach 593ec7b95c610fa68cec97926bc501d0e248230b1cf57c781f30d27f225e91dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:47:33.022 INFO:journalctl@ceph.osd.4.smithi174.stdout:Sep 20 18:47:32 smithi174 podman[68326]: 2024-09-20 18:47:32.947515129 +0000 UTC m=+0.524920059 container died 593ec7b95c610fa68cec97926bc501d0e248230b1cf57c781f30d27f225e91dd (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-20T18:47:33.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:33.314+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:33.522 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:33.314+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:33.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:33.477+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:33.522 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:33.477+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:33.582 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.4.service' 2024-09-20T18:47:33.622 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:33.623 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-20T18:47:33.623 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-20T18:47:33.623 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.5 2024-09-20T18:47:33.937 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:33 smithi174 systemd[1]: Stopping Ceph osd.5 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:47:33.938 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:33.604+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:33.938 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:33.604+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:34.271 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:33.920+0000 7f53d5fde640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:47:34.272 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:33.920+0000 7f53d5fde640 -1 osd.5 142 *** Got signal Terminated *** 2024-09-20T18:47:34.272 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:33 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:33.921+0000 7f53d5fde640 -1 osd.5 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:34.605 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:34 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:34.491+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:34.605 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:34 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:34.491+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:34.605 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:34 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:34.332+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:34.605 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:34 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:34.332+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:35.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:34 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:34.588+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:35.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:34 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:34.588+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:35.620 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:35 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:35.475+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:35.621 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:35 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:35.475+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:35.621 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:35 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:35.344+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:35.621 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:35 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:35.344+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:36.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:35 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:35.603+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:36.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:35 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:35.603+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:36.657 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:36.442+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:36.657 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:36.442+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:36.657 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:36.442+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:12.134226+0000 front 2024-09-20T18:47:12.134260+0000 (oldest deadline 2024-09-20T18:47:36.234070+0000) 2024-09-20T18:47:36.657 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:36.322+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:36.658 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:36.322+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:36.658 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:36.322+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:37.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:36.640+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:37.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:36 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:36.640+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:37.692 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:37.407+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:37.692 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:37.407+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:37.692 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:37.407+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:12.134226+0000 front 2024-09-20T18:47:12.134260+0000 (oldest deadline 2024-09-20T18:47:36.234070+0000) 2024-09-20T18:47:37.693 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:37.325+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:37.693 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:37.325+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:37.693 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:37.325+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:38.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:37.676+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:38.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:37 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:37.676+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:38.731 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:38.400+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:57.031849+0000 front 2024-09-20T18:46:57.031929+0000 (oldest deadline 2024-09-20T18:47:19.931819+0000) 2024-09-20T18:47:38.731 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:38.401+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:04.533240+0000 front 2024-09-20T18:47:04.533129+0000 (oldest deadline 2024-09-20T18:47:28.033220+0000) 2024-09-20T18:47:38.732 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5[55350]: 2024-09-20T18:47:38.401+0000 7f53d25f3640 -1 osd.5 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:12.134226+0000 front 2024-09-20T18:47:12.134260+0000 (oldest deadline 2024-09-20T18:47:36.234070+0000) 2024-09-20T18:47:38.732 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:38.338+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:38.732 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:38.338+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:38.732 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:38.338+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:39.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:38.715+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:39.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:38 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:38.715+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:39.703 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:39 smithi174 podman[68429]: 2024-09-20 18:47:39.478436605 +0000 UTC m=+5.649536797 container died b38346191a9dd318608fd671eeaa0830c2899de4848faf91b589f2344fe51566 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:47:39.704 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:39.359+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:39.704 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:39.359+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:39.704 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:39.359+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:40.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:39 smithi174 podman[68429]: 2024-09-20 18:47:39.820704118 +0000 UTC m=+5.991804309 container cleanup b38346191a9dd318608fd671eeaa0830c2899de4848faf91b589f2344fe51566 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-20T18:47:40.021 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:39 smithi174 bash[68429]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5 2024-09-20T18:47:40.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:39.686+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:40.021 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:39 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:39.686+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:40.271 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:40 smithi174 podman[68444]: 2024-09-20 18:47:40.005153299 +0000 UTC m=+0.523394565 container remove b38346191a9dd318608fd671eeaa0830c2899de4848faf91b589f2344fe51566 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-09-20T18:47:40.699 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:40 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:40.395+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:40.699 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:40 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:40.395+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:40.699 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:40 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:40.395+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:40.700 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:40 smithi174 podman[68533]: 2024-09-20 18:47:40.430955469 +0000 UTC m=+0.111058663 container create a9688027bc0d832187a6d7e847ea1db718712fe542cd892f36960c389357077d (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-20T18:47:40.700 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:40 smithi174 podman[68533]: 2024-09-20 18:47:40.335593296 +0000 UTC m=+0.015696512 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:40.700 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:40 smithi174 podman[68533]: 2024-09-20 18:47:40.683460228 +0000 UTC m=+0.363563439 container init a9688027bc0d832187a6d7e847ea1db718712fe542cd892f36960c389357077d (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True) 2024-09-20T18:47:41.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:40 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:40.725+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:41.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:40 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:40.725+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:41.022 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:40 smithi174 podman[68533]: 2024-09-20 18:47:40.686810658 +0000 UTC m=+0.366913860 container start a9688027bc0d832187a6d7e847ea1db718712fe542cd892f36960c389357077d (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-20T18:47:41.022 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:40 smithi174 podman[68533]: 2024-09-20 18:47:40.766814629 +0000 UTC m=+0.446917837 container attach a9688027bc0d832187a6d7e847ea1db718712fe542cd892f36960c389357077d (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:47:41.022 INFO:journalctl@ceph.osd.5.smithi174.stdout:Sep 20 18:47:40 smithi174 podman[68533]: 2024-09-20 18:47:40.87489873 +0000 UTC m=+0.555001942 container died a9688027bc0d832187a6d7e847ea1db718712fe542cd892f36960c389357077d (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:47:41.513 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.5.service' 2024-09-20T18:47:41.549 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:41.550 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-20T18:47:41.550 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-20T18:47:41.550 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.6 2024-09-20T18:47:41.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:41.444+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:41.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:41.444+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:41.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:41.444+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:41.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:41 smithi174 systemd[1]: Stopping Ceph osd.6 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:47:42.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:41.865+0000 7fd3c22e3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:47:42.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:41.865+0000 7fd3c22e3640 -1 osd.6 142 *** Got signal Terminated *** 2024-09-20T18:47:42.272 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:41.865+0000 7fd3c22e3640 -1 osd.6 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:42.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:41.760+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:42.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:41.760+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:42.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:41 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:41.760+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:42.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:42 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:42.466+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:42.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:42 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:42.466+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:42.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:42 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:42.466+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:43.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:42 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:42.758+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:43.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:42 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:42.758+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:43.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:42 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:42.758+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:43.771 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:43.428+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:43.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:43.428+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:43.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:43.428+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:44.271 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:43.760+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:44.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:43.760+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:44.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:43 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:43.760+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:44.768 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:44 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:44.429+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:44.768 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:44 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:44.429+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:44.768 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:44 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:44.429+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:44.768 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:44 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:44.429+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:18.589801+0000 front 2024-09-20T18:47:18.589836+0000 (oldest deadline 2024-09-20T18:47:43.889669+0000) 2024-09-20T18:47:45.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:44 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:44.752+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:45.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:44 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:44.752+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:45.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:44 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:44.752+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:45.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:45.467+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:45.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:45.467+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:45.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:45.467+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:45.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:45.467+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:18.589801+0000 front 2024-09-20T18:47:18.589836+0000 (oldest deadline 2024-09-20T18:47:43.889669+0000) 2024-09-20T18:47:46.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:45.755+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:46.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:45.755+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:46.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:45.755+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:46.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:45 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:45.755+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:46.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:46.456+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.186834+0000 front 2024-09-20T18:46:58.186888+0000 (oldest deadline 2024-09-20T18:47:21.086721+0000) 2024-09-20T18:47:46.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:46.456+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:05.187804+0000 front 2024-09-20T18:47:05.187748+0000 (oldest deadline 2024-09-20T18:47:30.388243+0000) 2024-09-20T18:47:46.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:46.456+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:10.388928+0000 front 2024-09-20T18:47:10.388858+0000 (oldest deadline 2024-09-20T18:47:36.288765+0000) 2024-09-20T18:47:46.772 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6[57487]: 2024-09-20T18:47:46.456+0000 7fd3be8f8640 -1 osd.6 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:18.589801+0000 front 2024-09-20T18:47:18.589836+0000 (oldest deadline 2024-09-20T18:47:43.889669+0000) 2024-09-20T18:47:47.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:46.799+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:47.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:46.799+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:47.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:46.799+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:47.272 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:46 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:46.799+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:47.565 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:47 smithi174 podman[68650]: 2024-09-20 18:47:47.282517539 +0000 UTC m=+5.515394696 container died b5b9f7c2d0417e6da94c9a3e415a478270d1278bf78953b45b04cce7877c3ab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-20T18:47:48.022 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:47 smithi174 podman[68650]: 2024-09-20 18:47:47.731491421 +0000 UTC m=+5.964368631 container cleanup b5b9f7c2d0417e6da94c9a3e415a478270d1278bf78953b45b04cce7877c3ab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-20T18:47:48.022 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:47 smithi174 bash[68650]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6 2024-09-20T18:47:48.023 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:47 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:47.767+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:48.023 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:47 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:47.767+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:48.023 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:47 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:47.767+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:48.023 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:47 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:47.767+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:48.413 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:48 smithi174 podman[68666]: 2024-09-20 18:47:48.050110573 +0000 UTC m=+0.765611774 container remove b5b9f7c2d0417e6da94c9a3e415a478270d1278bf78953b45b04cce7877c3ab1 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731) 2024-09-20T18:47:48.710 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:48 smithi174 podman[68754]: 2024-09-20 18:47:48.300401031 +0000 UTC m=+0.015652460 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:48.710 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:48 smithi174 podman[68754]: 2024-09-20 18:47:48.417357029 +0000 UTC m=+0.132608443 container create 6866d959defbefad4d2e74f8f5d7fbbd4c251624707effa3d37448980be0497f (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:47:48.710 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:48 smithi174 podman[68754]: 2024-09-20 18:47:48.611684113 +0000 UTC m=+0.326935539 container init 6866d959defbefad4d2e74f8f5d7fbbd4c251624707effa3d37448980be0497f (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-20T18:47:48.710 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:48 smithi174 podman[68754]: 2024-09-20 18:47:48.615059349 +0000 UTC m=+0.330310771 container start 6866d959defbefad4d2e74f8f5d7fbbd4c251624707effa3d37448980be0497f (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:47:48.711 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:48 smithi174 podman[68754]: 2024-09-20 18:47:48.694551467 +0000 UTC m=+0.409802899 container attach 6866d959defbefad4d2e74f8f5d7fbbd4c251624707effa3d37448980be0497f (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb) 2024-09-20T18:47:49.022 INFO:journalctl@ceph.osd.6.smithi174.stdout:Sep 20 18:47:48 smithi174 podman[68754]: 2024-09-20 18:47:48.804447072 +0000 UTC m=+0.519698500 container died 6866d959defbefad4d2e74f8f5d7fbbd4c251624707effa3d37448980be0497f (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-20T18:47:49.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:48 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:48.737+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:49.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:48 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:48.737+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:49.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:48 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:48.737+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:49.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:48 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:48.737+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:49.466 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.6.service' 2024-09-20T18:47:49.508 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:49.508 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-20T18:47:49.508 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-20T18:47:49.508 DEBUG:teuthology.orchestra.run.smithi174:> sudo systemctl stop ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.7 2024-09-20T18:47:49.739 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 systemd[1]: Stopping Ceph osd.7 for c01c9b48-777c-11ef-baf5-efdc52797490... 2024-09-20T18:47:50.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:49.721+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:50.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:49.721+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:50.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:49.721+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:50.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:49.721+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:50.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:49.833+0000 7f380ad5f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-20T18:47:50.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:49.833+0000 7f380ad5f640 -1 osd.7 142 *** Got signal Terminated *** 2024-09-20T18:47:50.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:49 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:49.833+0000 7f380ad5f640 -1 osd.7 142 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-20T18:47:51.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:50.725+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:51.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:50.725+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:51.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:50.725+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:51.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:50 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:50.725+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:52.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:51.701+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:52.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:51.701+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:52.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:51.701+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:52.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:51.701+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:52.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:51 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:51.701+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.174:6804 osd.4 since back 2024-09-20T18:47:29.977725+0000 front 2024-09-20T18:47:29.977748+0000 (oldest deadline 2024-09-20T18:47:51.677772+0000) 2024-09-20T18:47:53.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:52 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:52.668+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:53.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:52 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:52.668+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:53.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:52 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:52.668+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:53.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:52 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:52.668+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:53.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:52 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:52.668+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.174:6804 osd.4 since back 2024-09-20T18:47:29.977725+0000 front 2024-09-20T18:47:29.977748+0000 (oldest deadline 2024-09-20T18:47:51.677772+0000) 2024-09-20T18:47:54.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:53 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:53.694+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:54.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:53 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:53.694+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:54.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:53 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:53.694+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:54.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:53 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:53.694+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:54.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:53 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:53.694+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.174:6804 osd.4 since back 2024-09-20T18:47:29.977725+0000 front 2024-09-20T18:47:29.977748+0000 (oldest deadline 2024-09-20T18:47:51.677772+0000) 2024-09-20T18:47:55.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:54 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:54.708+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6806 osd.0 since back 2024-09-20T18:46:58.073238+0000 front 2024-09-20T18:46:58.073352+0000 (oldest deadline 2024-09-20T18:47:20.973072+0000) 2024-09-20T18:47:55.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:54 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:54.708+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6814 osd.1 since back 2024-09-20T18:47:03.174452+0000 front 2024-09-20T18:47:03.174377+0000 (oldest deadline 2024-09-20T18:47:31.374781+0000) 2024-09-20T18:47:55.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:54 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:54.708+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6822 osd.2 since back 2024-09-20T18:47:11.375288+0000 front 2024-09-20T18:47:11.375383+0000 (oldest deadline 2024-09-20T18:47:40.775606+0000) 2024-09-20T18:47:55.022 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:54 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:54.708+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.12:6830 osd.3 since back 2024-09-20T18:47:20.776240+0000 front 2024-09-20T18:47:20.776263+0000 (oldest deadline 2024-09-20T18:47:45.476123+0000) 2024-09-20T18:47:55.023 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:54 smithi174 ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7[59608]: 2024-09-20T18:47:54.708+0000 7f380336c640 -1 osd.7 142 heartbeat_check: no reply from 172.21.15.174:6804 osd.4 since back 2024-09-20T18:47:29.977725+0000 front 2024-09-20T18:47:29.977748+0000 (oldest deadline 2024-09-20T18:47:51.677772+0000) 2024-09-20T18:47:55.509 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:55 smithi174 podman[68858]: 2024-09-20 18:47:55.249721331 +0000 UTC m=+5.525396449 container died cc57872beb577df28d72d92cb92ee1e15a6848b05b3b859f1e79253f2b386e79 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-20T18:47:55.771 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:55 smithi174 podman[68858]: 2024-09-20 18:47:55.65066837 +0000 UTC m=+5.926343518 container cleanup cc57872beb577df28d72d92cb92ee1e15a6848b05b3b859f1e79253f2b386e79 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:47:55.772 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:55 smithi174 bash[68858]: ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7 2024-09-20T18:47:56.204 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:55 smithi174 podman[68886]: 2024-09-20 18:47:55.876940319 +0000 UTC m=+0.628147551 container remove cc57872beb577df28d72d92cb92ee1e15a6848b05b3b859f1e79253f2b386e79 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-20T18:47:56.512 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:56 smithi174 podman[68973]: 2024-09-20 18:47:56.09099429 +0000 UTC m=+0.015436001 image pull 441f9a3fe51528ef653b21f42e9d884db6124a0c7d482e865382bdfdb9ec979c quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2 2024-09-20T18:47:56.512 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:56 smithi174 podman[68973]: 2024-09-20 18:47:56.194077539 +0000 UTC m=+0.118519237 container create e3dd0282b6cc6899f03c35587031fb726adf0677261001c168c2d19bce8520e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-20T18:47:56.512 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:56 smithi174 podman[68973]: 2024-09-20 18:47:56.413412421 +0000 UTC m=+0.337854119 container init e3dd0282b6cc6899f03c35587031fb726adf0677261001c168c2d19bce8520e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:47:56.512 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:56 smithi174 podman[68973]: 2024-09-20 18:47:56.416821324 +0000 UTC m=+0.341263024 container start e3dd0282b6cc6899f03c35587031fb726adf0677261001c168c2d19bce8520e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-20T18:47:56.513 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:56 smithi174 podman[68973]: 2024-09-20 18:47:56.496614052 +0000 UTC m=+0.421055756 container attach e3dd0282b6cc6899f03c35587031fb726adf0677261001c168c2d19bce8520e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-20T18:47:56.772 INFO:journalctl@ceph.osd.7.smithi174.stdout:Sep 20 18:47:56 smithi174 podman[68973]: 2024-09-20 18:47:56.605462787 +0000 UTC m=+0.529904482 container died e3dd0282b6cc6899f03c35587031fb726adf0677261001c168c2d19bce8520e8 (image=quay.ceph.io/ceph-ci/ceph@sha256:8a1a6edbc97129e01d4344a6578add33aaab808082c84049bd2e48e1f97b80d2, name=ceph-c01c9b48-777c-11ef-baf5-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-20-1028-88deadb, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0) 2024-09-20T18:47:57.185 DEBUG:teuthology.orchestra.run.smithi174:> sudo pkill -f 'journalctl -f -n 0 -u ceph-c01c9b48-777c-11ef-baf5-efdc52797490@osd.7.service' 2024-09-20T18:47:57.222 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-20T18:47:57.223 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-20T18:47:57.223 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c01c9b48-777c-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-20T18:50:56.669 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c01c9b48-777c-11ef-baf5-efdc52797490 --force --keep-logs 2024-09-20T18:51:16.190 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T18:51:16.225 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-20T18:51:16.252 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-20T18:51:16.253 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659/remote/smithi012/crash 2024-09-20T18:51:16.254 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/crash -- . 2024-09-20T18:51:16.348 INFO:teuthology.orchestra.run.smithi012.stderr:tar: /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-20T18:51:16.348 INFO:teuthology.orchestra.run.smithi012.stderr:tar: Error is not recoverable: exiting now 2024-09-20T18:51:16.350 DEBUG:teuthology.misc:Transferring archived files from smithi174:/var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/crash to /home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659/remote/smithi174/crash 2024-09-20T18:51:16.351 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/crash -- . 2024-09-20T18:51:16.398 INFO:teuthology.orchestra.run.smithi174.stderr:tar: /var/lib/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/crash: Cannot open: No such file or directory 2024-09-20T18:51:16.398 INFO:teuthology.orchestra.run.smithi174.stderr:tar: Error is not recoverable: exiting now 2024-09-20T18:51:16.399 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-20T18:51:16.399 DEBUG:teuthology.orchestra.run.smithi012:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-20T18:51:16.430 INFO:tasks.cephadm:Compressing logs... 2024-09-20T18:51:16.430 DEBUG:teuthology.orchestra.run.smithi012:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-20T18:51:16.473 DEBUG:teuthology.orchestra.run.smithi174:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-20T18:51:16.497 INFO:teuthology.orchestra.run.smithi174.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.7.log 2024-09-20T18:51:16.497 INFO:teuthology.orchestra.run.smithi174.stderr:: No such file or directory 2024-09-20T18:51:16.498 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-20T18:51:16.498 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.4.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.0.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:: No such file or directory 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mon.smithi012.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.1.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mon.smithi012.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-volume.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mgr.smithi012.xkxxrd.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mgr.smithi012.xkxxrd.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi012.jjsnmc.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi012.jjsnmc.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.2.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr: 88.9% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi012.jjsnmc.log.gz 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.log 2024-09-20T18:51:16.504 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.2.log: 85.9% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-20T18:51:16.505 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.3.log 2024-09-20T18:51:16.506 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.audit.log 2024-09-20T18:51:16.506 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mgr.smithi174.nxkptu.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.cephadm.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.audit.log: /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mgr.smithi174.nxkptu.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi174.zwzgjx.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.6.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi174.zwzgjx.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.5.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr: 85.8% 87.7% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mds.foofs.smithi174.zwzgjx.log.gz 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr: -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.cephadm.log.gz 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mon.smithi174.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.log: 91.6% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-20T18:51:16.507 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mon.smithi174.log: 87.7% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-20T18:51:16.508 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.log: 91.5% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.audit.log.gz 2024-09-20T18:51:16.508 INFO:teuthology.orchestra.run.smithi012.stderr: 87.7% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph.log.gz 2024-09-20T18:51:16.508 INFO:teuthology.orchestra.run.smithi012.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-20T18:51:16.513 INFO:teuthology.orchestra.run.smithi174.stderr:/var/log/ceph/cephadm.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-20T18:51:16.516 INFO:teuthology.orchestra.run.smithi174.stderr: 91.6% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-20T18:51:16.520 INFO:teuthology.orchestra.run.smithi012.stderr:/var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.5% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-volume.log.gz 2024-09-20T18:51:16.550 INFO:teuthology.orchestra.run.smithi012.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-20T18:51:16.551 INFO:teuthology.orchestra.run.smithi174.stderr: 89.8% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mgr.smithi174.nxkptu.log.gz 2024-09-20T18:51:16.813 INFO:teuthology.orchestra.run.smithi012.stderr: 88.9% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mgr.smithi012.xkxxrd.log.gz 2024-09-20T18:51:17.244 INFO:teuthology.orchestra.run.smithi174.stderr: 93.1% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mon.smithi174.log.gz 2024-09-20T18:51:17.464 INFO:teuthology.orchestra.run.smithi012.stderr: 90.7% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-mon.smithi012.log.gz 2024-09-20T18:51:29.410 INFO:teuthology.orchestra.run.smithi012.stderr: 94.4% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.2.log.gz 2024-09-20T18:51:29.881 INFO:teuthology.orchestra.run.smithi174.stderr: 93.9% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.4.log.gz 2024-09-20T18:51:31.588 INFO:teuthology.orchestra.run.smithi174.stderr: 94.2% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.6.log.gz 2024-09-20T18:51:32.483 INFO:teuthology.orchestra.run.smithi012.stderr: 93.9% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.0.log.gz 2024-09-20T18:51:34.192 INFO:teuthology.orchestra.run.smithi012.stderr: 94.1% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.1.log.gz 2024-09-20T18:51:35.060 INFO:teuthology.orchestra.run.smithi012.stderr: 93.8% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.3.log.gz 2024-09-20T18:51:35.062 INFO:teuthology.orchestra.run.smithi012.stderr: 2024-09-20T18:51:35.062 INFO:teuthology.orchestra.run.smithi012.stderr:real 0m18.575s 2024-09-20T18:51:35.063 INFO:teuthology.orchestra.run.smithi012.stderr:user 1m4.040s 2024-09-20T18:51:35.063 INFO:teuthology.orchestra.run.smithi012.stderr:sys 0m2.340s 2024-09-20T18:51:35.360 INFO:teuthology.orchestra.run.smithi174.stderr: 94.1% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.5.log.gz 2024-09-20T18:51:37.907 INFO:teuthology.orchestra.run.smithi174.stderr: 94.0% -- replaced with /var/log/ceph/c01c9b48-777c-11ef-baf5-efdc52797490/ceph-osd.7.log.gz 2024-09-20T18:51:37.908 INFO:teuthology.orchestra.run.smithi174.stderr: 2024-09-20T18:51:37.908 INFO:teuthology.orchestra.run.smithi174.stderr:real 0m21.421s 2024-09-20T18:51:37.909 INFO:teuthology.orchestra.run.smithi174.stderr:user 1m6.839s 2024-09-20T18:51:37.909 INFO:teuthology.orchestra.run.smithi174.stderr:sys 0m2.607s 2024-09-20T18:51:37.909 INFO:tasks.cephadm:Archiving logs... 2024-09-20T18:51:37.910 DEBUG:teuthology.misc:Transferring archived files from smithi012:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659/remote/smithi012/log 2024-09-20T18:51:37.911 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-20T18:51:46.229 DEBUG:teuthology.misc:Transferring archived files from smithi174:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659/remote/smithi174/log 2024-09-20T18:51:46.230 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-20T18:51:55.085 INFO:tasks.cephadm:Removing cluster... 2024-09-20T18:51:55.085 DEBUG:teuthology.orchestra.run.smithi012:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c01c9b48-777c-11ef-baf5-efdc52797490 --force 2024-09-20T18:51:55.457 DEBUG:teuthology.orchestra.run.smithi174:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid c01c9b48-777c-11ef-baf5-efdc52797490 --force 2024-09-20T18:51:55.821 INFO:tasks.cephadm:Removing cephadm ... 2024-09-20T18:51:55.821 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-20T18:51:55.838 DEBUG:teuthology.orchestra.run.smithi174:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-20T18:51:55.854 INFO:tasks.cephadm:Teardown complete 2024-09-20T18:51:55.854 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-20T18:51:55.866 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-20T18:51:55.867 DEBUG:teuthology.orchestra.run.smithi012:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T18:51:55.881 DEBUG:teuthology.orchestra.run.smithi174:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-20T18:51:55.894 INFO:teuthology.orchestra.run.smithi012.stderr:bash: line 1: ntpq: command not found 2024-09-20T18:51:55.912 INFO:teuthology.orchestra.run.smithi012.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T18:51:55.912 INFO:teuthology.orchestra.run.smithi012.stdout:=============================================================================== 2024-09-20T18:51:55.912 INFO:teuthology.orchestra.run.smithi012.stdout:^* hv01.front.sepia.ceph.com 3 6 377 50 -103us[ -97us] +/- 61ms 2024-09-20T18:51:55.912 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 51 +1608us[+1614us] +/- 62ms 2024-09-20T18:51:55.912 INFO:teuthology.orchestra.run.smithi012.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 -1641us[-1635us] +/- 68ms 2024-09-20T18:51:55.912 INFO:teuthology.orchestra.run.smithi012.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:51:55.924 INFO:teuthology.orchestra.run.smithi174.stderr:bash: line 1: ntpq: command not found 2024-09-20T18:51:55.928 INFO:teuthology.orchestra.run.smithi174.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-20T18:51:55.928 INFO:teuthology.orchestra.run.smithi174.stdout:=============================================================================== 2024-09-20T18:51:55.928 INFO:teuthology.orchestra.run.smithi174.stdout:^* hv01.front.sepia.ceph.com 3 6 377 54 -106us[ -84us] +/- 61ms 2024-09-20T18:51:55.928 INFO:teuthology.orchestra.run.smithi174.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 55 +1615us[+1637us] +/- 62ms 2024-09-20T18:51:55.928 INFO:teuthology.orchestra.run.smithi174.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 55 -1648us[-1625us] +/- 68ms 2024-09-20T18:51:55.928 INFO:teuthology.orchestra.run.smithi174.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-20T18:51:55.929 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-20T18:51:55.938 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-20T18:51:55.939 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-20T18:51:55.953 DEBUG:teuthology.orchestra.run.smithi012:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-20T18:51:56.034 DEBUG:teuthology.orchestra.run.smithi174:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-20T18:51:56.116 DEBUG:teuthology.orchestra.run.smithi012:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T18:51:56.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:51:56.146 DEBUG:teuthology.orchestra.run.smithi174:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-20T18:51:56.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:51:56.174 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-20T18:51:56.186 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-20T18:51:56.204 INFO:teuthology.task.internal:Duration was 2246.431621 seconds 2024-09-20T18:51:56.205 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-20T18:51:56.220 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-20T18:51:56.220 DEBUG:teuthology.orchestra.run.smithi012:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-20T18:51:56.222 DEBUG:teuthology.orchestra.run.smithi174:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-20T18:51:56.261 INFO:teuthology.orchestra.run.smithi174.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T18:51:56.262 INFO:teuthology.orchestra.run.smithi012.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-20T18:51:56.773 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-20T18:51:56.773 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi012.front.sepia.ceph.com 2024-09-20T18:51:56.773 DEBUG:teuthology.orchestra.run.smithi012:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-20T18:51:56.837 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi174.front.sepia.ceph.com 2024-09-20T18:51:56.838 DEBUG:teuthology.orchestra.run.smithi174:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-20T18:51:56.873 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-20T18:51:56.873 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-20T18:51:56.880 DEBUG:teuthology.orchestra.run.smithi174:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-20T18:51:56.938 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-20T18:51:56.939 DEBUG:teuthology.orchestra.run.smithi012:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-20T18:51:56.952 DEBUG:teuthology.orchestra.run.smithi174:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-20T18:52:11.528 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-20T18:52:11.548 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-20T18:52:11.548 DEBUG:teuthology.orchestra.run.smithi012:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-20T18:52:11.575 DEBUG:teuthology.orchestra.run.smithi174:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-20T18:52:11.606 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-20T18:52:11.616 DEBUG:teuthology.orchestra.run.smithi012:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-20T18:52:11.619 DEBUG:teuthology.orchestra.run.smithi174:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-20T18:52:11.640 INFO:teuthology.orchestra.run.smithi012.stdout:kernel.core_pattern = core 2024-09-20T18:52:11.673 INFO:teuthology.orchestra.run.smithi174.stdout:kernel.core_pattern = core 2024-09-20T18:52:11.780 DEBUG:teuthology.orchestra.run.smithi012:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-20T18:52:11.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:52:11.795 DEBUG:teuthology.orchestra.run.smithi174:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-20T18:52:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-20T18:52:11.837 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-20T18:52:11.848 INFO:teuthology.task.internal:Transferring archived files... 2024-09-20T18:52:11.849 DEBUG:teuthology.misc:Transferring archived files from smithi012:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659/remote/smithi012 2024-09-20T18:52:11.849 DEBUG:teuthology.orchestra.run.smithi012:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-20T18:52:11.912 DEBUG:teuthology.misc:Transferring archived files from smithi174:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-20_12:24:58-orch:cephadm-wip-guits-main-2024-09-20-1028-distro-default-smithi/7912659/remote/smithi174 2024-09-20T18:52:11.912 DEBUG:teuthology.orchestra.run.smithi174:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-20T18:52:11.952 INFO:teuthology.task.internal:Removing archive directory... 2024-09-20T18:52:11.953 DEBUG:teuthology.orchestra.run.smithi012:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-20T18:52:11.955 DEBUG:teuthology.orchestra.run.smithi174:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-20T18:52:12.003 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-20T18:52:12.014 INFO:teuthology.task.internal:Not uploading archives. 2024-09-20T18:52:12.014 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-20T18:52:12.036 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-20T18:52:12.036 DEBUG:teuthology.orchestra.run.smithi012:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-20T18:52:12.038 DEBUG:teuthology.orchestra.run.smithi174:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-20T18:52:12.051 INFO:teuthology.orchestra.run.smithi012.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 20 18:52 /home/ubuntu/cephtest 2024-09-20T18:52:12.059 INFO:teuthology.orchestra.run.smithi174.stdout: 265644 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 20 18:52 /home/ubuntu/cephtest 2024-09-20T18:52:12.060 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-20T18:52:12.118 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-20T18:52:12.147 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2246.4316205978394 owner: scheduled_gabrioux@teuthology success: true 2024-09-20T18:52:12.147 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-20T18:52:12.231 INFO:teuthology.run:pass