2024-09-13T05:11:02.159 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T05:11:02.177 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T05:11:02.275 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116 branch: wip-guits-main-2024-09-12-1245 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7903116' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-12-1245 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-12-1245 sha1: 9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 1848 sha1: 9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-12-1245 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 targets: smithi053.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO2ls/YddihqSb63RP0J0Tkb6GwkI4+8ziuRmAjO5U70ZS51wocA/899ldM2yrFf4ItXTTQA83bqiuPTvsc3ewA= smithi110.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCjPYp69ZTJIdPMgBdqKwkX9fc1gQbteI0S8lk6wlV3g511pKCTQgj4A4t8Q1qhQH7jgH5bjBe1HyB2sv+Ab8nU= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_05:00:55 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-13T05:11:02.275 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3/qa; will attempt to use it 2024-09-13T05:11:02.276 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3/qa/tasks 2024-09-13T05:11:02.276 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T05:11:02.278 INFO:teuthology.task.internal:Checking packages... 2024-09-13T05:11:02.300 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3' 2024-09-13T05:11:02.300 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T05:11:02.300 INFO:teuthology.packaging:ref: None 2024-09-13T05:11:02.300 INFO:teuthology.packaging:tag: None 2024-09-13T05:11:02.300 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-12-1245 2024-09-13T05:11:02.300 INFO:teuthology.packaging:sha1: 9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 2024-09-13T05:11:02.300 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-12-1245 2024-09-13T05:11:02.454 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4911.g9d608eb1 2024-09-13T05:11:02.456 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T05:11:02.462 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T05:11:02.462 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T05:11:02.472 INFO:teuthology.task.internal:Saving configuration 2024-09-13T05:11:02.486 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T05:11:02.498 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T05:11:02.523 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi053.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 05:06:13.847273', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO2ls/YddihqSb63RP0J0Tkb6GwkI4+8ziuRmAjO5U70ZS51wocA/899ldM2yrFf4ItXTTQA83bqiuPTvsc3ewA='} 2024-09-13T05:11:02.552 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi110.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 05:06:13.845989', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCjPYp69ZTJIdPMgBdqKwkX9fc1gQbteI0S8lk6wlV3g511pKCTQgj4A4t8Q1qhQH7jgH5bjBe1HyB2sv+Ab8nU='} 2024-09-13T05:11:02.552 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T05:11:02.561 INFO:teuthology.task.internal:roles: ubuntu@smithi053.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-13T05:11:02.561 INFO:teuthology.task.internal:roles: ubuntu@smithi110.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-13T05:11:02.561 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T05:11:02.650 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fcb58bb3be0>, signals=[15]) 2024-09-13T05:11:02.650 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T05:11:02.661 INFO:teuthology.task.internal:Opening connections... 2024-09-13T05:11:02.662 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:11:02.663 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:11:02.746 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi110.front.sepia.ceph.com 2024-09-13T05:11:02.747 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:11:02.829 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T05:11:02.838 DEBUG:teuthology.orchestra.run.smithi053:> uname -m 2024-09-13T05:11:02.856 INFO:teuthology.orchestra.run.smithi053.stdout:x86_64 2024-09-13T05:11:02.856 DEBUG:teuthology.orchestra.run.smithi053:> cat /etc/os-release 2024-09-13T05:11:02.914 INFO:teuthology.orchestra.run.smithi053.stdout:NAME="CentOS Stream" 2024-09-13T05:11:02.914 INFO:teuthology.orchestra.run.smithi053.stdout:VERSION="9" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:ID="centos" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:ID_LIKE="rhel fedora" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:VERSION_ID="9" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:PLATFORM_ID="platform:el9" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:ANSI_COLOR="0;31" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:LOGO="fedora-logo-icon" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:HOME_URL="https://centos.org/" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T05:11:02.915 INFO:teuthology.orchestra.run.smithi053.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T05:11:02.916 INFO:teuthology.lock.ops:Updating smithi053.front.sepia.ceph.com on lock server 2024-09-13T05:11:02.942 DEBUG:teuthology.orchestra.run.smithi110:> uname -m 2024-09-13T05:11:02.961 INFO:teuthology.orchestra.run.smithi110.stdout:x86_64 2024-09-13T05:11:02.961 DEBUG:teuthology.orchestra.run.smithi110:> cat /etc/os-release 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:NAME="CentOS Stream" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION="9" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:ID="centos" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:ID_LIKE="rhel fedora" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:VERSION_ID="9" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:PLATFORM_ID="platform:el9" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:ANSI_COLOR="0;31" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:LOGO="fedora-logo-icon" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:HOME_URL="https://centos.org/" 2024-09-13T05:11:03.018 INFO:teuthology.orchestra.run.smithi110.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T05:11:03.019 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T05:11:03.019 INFO:teuthology.orchestra.run.smithi110.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T05:11:03.019 INFO:teuthology.lock.ops:Updating smithi110.front.sepia.ceph.com on lock server 2024-09-13T05:11:03.051 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T05:11:03.062 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T05:11:03.074 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T05:11:03.074 DEBUG:teuthology.orchestra.run.smithi053:> test '!' -e /home/ubuntu/cephtest 2024-09-13T05:11:03.077 DEBUG:teuthology.orchestra.run.smithi110:> test '!' -e /home/ubuntu/cephtest 2024-09-13T05:11:03.093 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T05:11:03.104 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T05:11:03.104 DEBUG:teuthology.orchestra.run.smithi053:> test -z $(ls -A /var/lib/ceph) 2024-09-13T05:11:03.134 DEBUG:teuthology.orchestra.run.smithi110:> test -z $(ls -A /var/lib/ceph) 2024-09-13T05:11:03.161 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T05:11:03.239 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T05:11:03.258 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T05:11:03.259 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T05:11:03.259 DEBUG:teuthology.orchestra.run.smithi053:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T05:11:03.259 DEBUG:teuthology.orchestra.run.smithi110:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T05:11:03.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:11:03.276 DEBUG:teuthology.orchestra.run.smithi110:> uname -r 2024-09-13T05:11:03.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:11:03.277 DEBUG:teuthology.orchestra.run.smithi053:> uname -r 2024-09-13T05:11:03.333 INFO:teuthology.orchestra.run.smithi110.stdout:5.14.0-503.el9.x86_64 2024-09-13T05:11:03.333 INFO:teuthology.task.kernel:Running kernel on smithi110: 5.14.0-503.el9.x86_64 2024-09-13T05:11:03.333 DEBUG:teuthology.orchestra.run.smithi110:> sudo yum install -y kernel 2024-09-13T05:11:03.334 INFO:teuthology.orchestra.run.smithi053.stdout:5.14.0-503.el9.x86_64 2024-09-13T05:11:03.335 INFO:teuthology.task.kernel:Running kernel on smithi053: 5.14.0-503.el9.x86_64 2024-09-13T05:11:03.335 DEBUG:teuthology.orchestra.run.smithi053:> sudo yum install -y kernel 2024-09-13T05:11:05.650 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.2 MB 00:00 2024-09-13T05:11:07.635 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - BaseOS 3.3 MB/s | 8.2 MB 00:02 2024-09-13T05:11:09.992 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - AppStream 6.4 MB/s | 20 MB 00:03 2024-09-13T05:11:14.618 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - CRB 9.4 MB/s | 6.5 MB 00:00 2024-09-13T05:11:15.169 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - AppStream 3.2 MB/s | 20 MB 00:06 2024-09-13T05:11:16.269 INFO:teuthology.orchestra.run.smithi053.stdout:CentOS Stream 9 - Extras packages 47 kB/s | 19 kB 00:00 2024-09-13T05:11:17.307 INFO:teuthology.orchestra.run.smithi053.stdout:Extra Packages for Enterprise Linux 25 MB/s | 23 MB 00:00 2024-09-13T05:11:20.744 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 6.5 MB 00:01 2024-09-13T05:11:22.353 INFO:teuthology.orchestra.run.smithi110.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 19 kB 00:00 2024-09-13T05:11:22.970 INFO:teuthology.orchestra.run.smithi053.stdout:lab-extras 38 kB/s | 1.7 kB 00:00 2024-09-13T05:11:23.549 INFO:teuthology.orchestra.run.smithi110.stdout:Extra Packages for Enterprise Linux 21 MB/s | 23 MB 00:01 2024-09-13T05:11:25.036 INFO:teuthology.orchestra.run.smithi053.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T05:11:25.036 INFO:teuthology.orchestra.run.smithi053.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T05:11:25.112 INFO:teuthology.orchestra.run.smithi053.stdout:Dependencies resolved. 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout: Package Architecture Version Repository Size 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout:Installing: 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout:Installing dependencies: 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T05:11:25.117 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:11:25.118 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction Summary 2024-09-13T05:11:25.118 INFO:teuthology.orchestra.run.smithi053.stdout:================================================================================ 2024-09-13T05:11:25.118 INFO:teuthology.orchestra.run.smithi053.stdout:Install 4 Packages 2024-09-13T05:11:25.118 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:11:25.118 INFO:teuthology.orchestra.run.smithi053.stdout:Total download size: 79 M 2024-09-13T05:11:25.118 INFO:teuthology.orchestra.run.smithi053.stdout:Installed size: 126 M 2024-09-13T05:11:25.118 INFO:teuthology.orchestra.run.smithi053.stdout:Downloading Packages: 2024-09-13T05:11:25.347 INFO:teuthology.orchestra.run.smithi053.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 217 kB/s | 23 kB 00:00 2024-09-13T05:11:25.731 INFO:teuthology.orchestra.run.smithi053.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 32 MB/s | 16 MB 00:00 2024-09-13T05:11:26.164 INFO:teuthology.orchestra.run.smithi053.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 37 MB/s | 35 MB 00:00 2024-09-13T05:11:26.431 INFO:teuthology.orchestra.run.smithi053.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 26 MB/s | 29 MB 00:01 2024-09-13T05:11:26.432 INFO:teuthology.orchestra.run.smithi053.stdout:-------------------------------------------------------------------------------- 2024-09-13T05:11:26.432 INFO:teuthology.orchestra.run.smithi053.stdout:Total 60 MB/s | 79 MB 00:01 2024-09-13T05:11:27.144 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction check 2024-09-13T05:11:27.525 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction check succeeded. 2024-09-13T05:11:27.525 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction test 2024-09-13T05:11:28.528 INFO:teuthology.orchestra.run.smithi053.stdout:Transaction test succeeded. 2024-09-13T05:11:28.528 INFO:teuthology.orchestra.run.smithi053.stdout:Running transaction 2024-09-13T05:11:28.981 INFO:teuthology.orchestra.run.smithi110.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-13T05:11:31.003 INFO:teuthology.orchestra.run.smithi110.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T05:11:31.003 INFO:teuthology.orchestra.run.smithi110.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T05:11:31.090 INFO:teuthology.orchestra.run.smithi110.stdout:Dependencies resolved. 2024-09-13T05:11:31.093 INFO:teuthology.orchestra.run.smithi053.stdout: Preparing : 1/1 2024-09-13T05:11:31.097 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-13T05:11:31.097 INFO:teuthology.orchestra.run.smithi110.stdout: Package Architecture Version Repository Size 2024-09-13T05:11:31.097 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-13T05:11:31.097 INFO:teuthology.orchestra.run.smithi110.stdout:Installing: 2024-09-13T05:11:31.097 INFO:teuthology.orchestra.run.smithi110.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout:Installing dependencies: 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction Summary 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout:================================================================================ 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout:Install 4 Packages 2024-09-13T05:11:31.098 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:11:31.099 INFO:teuthology.orchestra.run.smithi110.stdout:Total download size: 79 M 2024-09-13T05:11:31.099 INFO:teuthology.orchestra.run.smithi110.stdout:Installed size: 126 M 2024-09-13T05:11:31.099 INFO:teuthology.orchestra.run.smithi110.stdout:Downloading Packages: 2024-09-13T05:11:31.232 INFO:teuthology.orchestra.run.smithi110.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 263 kB/s | 23 kB 00:00 2024-09-13T05:11:31.615 INFO:teuthology.orchestra.run.smithi110.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-13T05:11:31.768 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T05:11:31.857 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T05:11:32.057 INFO:teuthology.orchestra.run.smithi110.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 38 MB/s | 35 MB 00:00 2024-09-13T05:11:32.658 INFO:teuthology.orchestra.run.smithi110.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 20 MB/s | 29 MB 00:01 2024-09-13T05:11:32.658 INFO:teuthology.orchestra.run.smithi110.stdout:-------------------------------------------------------------------------------- 2024-09-13T05:11:32.658 INFO:teuthology.orchestra.run.smithi110.stdout:Total 50 MB/s | 79 MB 00:01 2024-09-13T05:11:33.337 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction check 2024-09-13T05:11:33.695 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction check succeeded. 2024-09-13T05:11:33.696 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction test 2024-09-13T05:11:34.636 INFO:teuthology.orchestra.run.smithi110.stdout:Transaction test succeeded. 2024-09-13T05:11:34.637 INFO:teuthology.orchestra.run.smithi110.stdout:Running transaction 2024-09-13T05:11:34.639 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T05:11:34.778 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T05:11:37.085 INFO:teuthology.orchestra.run.smithi110.stdout: Preparing : 1/1 2024-09-13T05:11:37.758 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T05:11:37.834 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T05:11:39.970 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T05:11:40.012 INFO:teuthology.orchestra.run.smithi053.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:11:40.614 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T05:11:40.745 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T05:11:45.156 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:11:45.568 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T05:11:45.612 INFO:teuthology.orchestra.run.smithi110.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:11:50.442 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:24.718 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:24.759 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:25.021 INFO:teuthology.orchestra.run.smithi053.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:25.021 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T05:12:25.021 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T05:12:25.021 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T05:12:25.901 INFO:teuthology.orchestra.run.smithi053.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:25.901 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:12:25.901 INFO:teuthology.orchestra.run.smithi053.stdout:Installed: 2024-09-13T05:12:25.901 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T05:12:25.901 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T05:12:25.901 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T05:12:25.902 INFO:teuthology.orchestra.run.smithi053.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T05:12:25.902 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:12:25.902 INFO:teuthology.orchestra.run.smithi053.stdout:Complete! 2024-09-13T05:12:26.228 DEBUG:teuthology.orchestra.run.smithi053:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T05:12:26.321 INFO:teuthology.orchestra.run.smithi053.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T05:12:26.322 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T05:12:26.322 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T05:12:26.322 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T05:12:26.322 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T05:12:26.322 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-13T05:12:26.323 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi053.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T05:12:26.323 DEBUG:teuthology.orchestra.run.smithi053:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T05:12:26.382 INFO:teuthology.orchestra.run.smithi053.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T05:12:26.382 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T05:12:26.382 DEBUG:teuthology.orchestra.run.smithi053:> sudo rpm -qi grub2-tools 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Name : grub2-tools 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Epoch : 1 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Version : 2.06 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Release : 82.el9 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Architecture: x86_64 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Group : Unspecified 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Size : 8274670 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:License : GPLv3+ 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T05:12:26.414 INFO:teuthology.orchestra.run.smithi053.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:Packager : builder@centos.org 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:Vendor : CentOS 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:Summary : Support tools for GRUB. 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:Description : 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:hardware devices. 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:12:26.415 INFO:teuthology.orchestra.run.smithi053.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T05:12:26.416 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T05:12:26.417 INFO:teuthology.task.kernel:Updating grub on smithi053 to boot 5.14.0-505.el9.x86_64 2024-09-13T05:12:26.417 DEBUG:teuthology.orchestra.run.smithi053:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T05:12:27.051 INFO:teuthology.orchestra.run.smithi053.stderr:Generating grub configuration file ... 2024-09-13T05:12:28.376 INFO:teuthology.orchestra.run.smithi053.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T05:12:28.412 INFO:teuthology.orchestra.run.smithi053.stderr:done 2024-09-13T05:12:28.415 DEBUG:teuthology.orchestra.run.smithi053:> mktemp 2024-09-13T05:12:28.430 INFO:teuthology.orchestra.run.smithi053.stdout:/tmp/tmp.NetQ7L9bB7 2024-09-13T05:12:28.430 DEBUG:teuthology.orchestra.run.smithi053:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.NetQ7L9bB7 2024-09-13T05:12:28.496 DEBUG:teuthology.orchestra.run.smithi053:> sudo chmod 0666 /tmp/tmp.NetQ7L9bB7 2024-09-13T05:12:28.564 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:28.604 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:28.641 DEBUG:teuthology.orchestra.remote:smithi053:/tmp/tmp.NetQ7L9bB7 is 6KB 2024-09-13T05:12:28.663 DEBUG:teuthology.orchestra.run.smithi053:> rm -fr /tmp/tmp.NetQ7L9bB7 2024-09-13T05:12:28.677 DEBUG:teuthology.orchestra.run.smithi053:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T05:12:28.741 INFO:teuthology.orchestra.run.smithi053.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T05:12:28.741 INFO:teuthology.orchestra.run.smithi053.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T05:12:28.741 INFO:teuthology.orchestra.run.smithi053.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T05:12:28.741 INFO:teuthology.orchestra.run.smithi053.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T05:12:28.742 DEBUG:teuthology.orchestra.run.smithi053:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T05:12:28.879 INFO:teuthology.orchestra.run.smithi110.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:28.880 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T05:12:28.880 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T05:12:28.880 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T05:12:28.890 DEBUG:teuthology.orchestra.run.smithi053:> sudo shutdown -r now 2024-09-13T05:12:29.744 INFO:teuthology.orchestra.run.smithi110.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout:Installed: 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:12:29.745 INFO:teuthology.orchestra.run.smithi110.stdout:Complete! 2024-09-13T05:12:30.107 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T05:12:30.192 INFO:teuthology.orchestra.run.smithi110.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T05:12:30.192 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T05:12:30.192 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T05:12:30.192 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T05:12:30.192 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T05:12:30.192 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-13T05:12:30.193 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi110.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T05:12:30.193 DEBUG:teuthology.orchestra.run.smithi110:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T05:12:30.251 INFO:teuthology.orchestra.run.smithi110.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T05:12:30.251 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T05:12:30.251 DEBUG:teuthology.orchestra.run.smithi110:> sudo rpm -qi grub2-tools 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Name : grub2-tools 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Epoch : 1 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Version : 2.06 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Release : 82.el9 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Architecture: x86_64 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Group : Unspecified 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:Size : 8274670 2024-09-13T05:12:30.283 INFO:teuthology.orchestra.run.smithi110.stdout:License : GPLv3+ 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Packager : builder@centos.org 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Vendor : CentOS 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Summary : Support tools for GRUB. 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:Description : 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T05:12:30.284 INFO:teuthology.orchestra.run.smithi110.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T05:12:30.285 INFO:teuthology.orchestra.run.smithi110.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T05:12:30.285 INFO:teuthology.orchestra.run.smithi110.stdout:hardware devices. 2024-09-13T05:12:30.285 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:12:30.285 INFO:teuthology.orchestra.run.smithi110.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T05:12:30.286 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T05:12:30.286 INFO:teuthology.task.kernel:Updating grub on smithi110 to boot 5.14.0-505.el9.x86_64 2024-09-13T05:12:30.286 DEBUG:teuthology.orchestra.run.smithi110:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T05:12:30.931 INFO:teuthology.orchestra.run.smithi110.stderr:Generating grub configuration file ... 2024-09-13T05:12:32.382 INFO:teuthology.orchestra.run.smithi110.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T05:12:32.412 INFO:teuthology.orchestra.run.smithi110.stderr:done 2024-09-13T05:12:32.422 DEBUG:teuthology.orchestra.run.smithi110:> mktemp 2024-09-13T05:12:32.437 INFO:teuthology.orchestra.run.smithi110.stdout:/tmp/tmp.Om4vVoykNR 2024-09-13T05:12:32.437 DEBUG:teuthology.orchestra.run.smithi110:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Om4vVoykNR 2024-09-13T05:12:32.504 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 0666 /tmp/tmp.Om4vVoykNR 2024-09-13T05:12:32.652 DEBUG:teuthology.orchestra.remote:smithi110:/tmp/tmp.Om4vVoykNR is 6KB 2024-09-13T05:12:32.700 DEBUG:teuthology.orchestra.run.smithi110:> rm -fr /tmp/tmp.Om4vVoykNR 2024-09-13T05:12:32.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T05:12:32.778 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T05:12:32.778 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T05:12:32.779 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T05:12:32.779 INFO:teuthology.orchestra.run.smithi110.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T05:12:32.780 DEBUG:teuthology.orchestra.run.smithi110:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T05:12:32.937 DEBUG:teuthology.orchestra.run.smithi110:> sudo shutdown -r now 2024-09-13T05:12:58.920 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-13T05:12:58.921 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-09-13T05:12:58.922 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:13:02.941 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-13T05:13:02.941 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-13T05:13:02.942 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:13:21.469 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.110 2024-09-13T05:13:30.478 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-13T05:13:30.480 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:13:58.961 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T05:14:07.970 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-09-13T05:14:07.972 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:14:08.310 DEBUG:teuthology.orchestra.run.smithi053:> true 2024-09-13T05:14:08.659 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-09-13T05:14:08.659 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T05:14:08.660 DEBUG:teuthology.orchestra.run.smithi053:> uname -r 2024-09-13T05:14:08.676 INFO:teuthology.orchestra.run.smithi053.stdout:5.14.0-505.el9.x86_64 2024-09-13T05:14:08.676 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T05:14:08.677 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T05:14:08.677 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T05:14:09.678 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-13T05:14:09.678 DEBUG:teuthology.orchestra.run.smithi053:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T05:14:09.734 INFO:teuthology.orchestra.run.smithi053.stdout:ttyS1 2024-09-13T05:14:09.758 DEBUG:teuthology.parallel:result is None 2024-09-13T05:14:30.508 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T05:14:42.509 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-13T05:14:42.510 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:14:42.909 DEBUG:teuthology.orchestra.run.smithi110:> true 2024-09-13T05:14:43.265 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-13T05:14:43.265 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T05:14:43.266 DEBUG:teuthology.orchestra.run.smithi110:> uname -r 2024-09-13T05:14:43.281 INFO:teuthology.orchestra.run.smithi110.stdout:5.14.0-505.el9.x86_64 2024-09-13T05:14:43.282 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T05:14:43.282 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T05:14:43.282 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T05:14:44.283 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-13T05:14:44.283 DEBUG:teuthology.orchestra.run.smithi110:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T05:14:44.357 INFO:teuthology.orchestra.run.smithi110.stdout:ttyS1 2024-09-13T05:14:44.377 DEBUG:teuthology.parallel:result is None 2024-09-13T05:14:44.377 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T05:14:44.385 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T05:14:44.385 DEBUG:teuthology.orchestra.run.smithi053:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T05:14:44.388 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T05:14:44.406 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T05:14:44.412 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T05:14:44.418 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T05:14:44.418 DEBUG:teuthology.orchestra.run.smithi053:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T05:14:44.448 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T05:14:44.479 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T05:14:44.485 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T05:14:44.485 DEBUG:teuthology.orchestra.run.smithi053:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T05:14:44.516 DEBUG:teuthology.orchestra.run.smithi110:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T05:14:44.552 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T05:14:44.552 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T05:14:44.563 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T05:14:44.564 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T05:14:44.566 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T05:14:44.573 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T05:14:44.573 DEBUG:teuthology.orchestra.run.smithi053:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T05:14:44.609 DEBUG:teuthology.orchestra.run.smithi110:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T05:14:44.636 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T05:14:44.645 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T05:14:44.646 DEBUG:teuthology.orchestra.run.smithi053:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T05:14:44.678 DEBUG:teuthology.orchestra.run.smithi110:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T05:14:44.694 DEBUG:teuthology.orchestra.run.smithi053:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T05:14:44.789 DEBUG:teuthology.orchestra.run.smithi053:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T05:14:44.830 DEBUG:teuthology.orchestra.run.smithi053:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T05:14:44.922 DEBUG:teuthology.orchestra.run.smithi053:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T05:14:44.956 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:14:44.956 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T05:14:45.023 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T05:14:45.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T05:14:45.123 DEBUG:teuthology.orchestra.run.smithi110:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T05:14:45.216 DEBUG:teuthology.orchestra.run.smithi110:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T05:14:45.249 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-13T05:14:45.249 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T05:14:45.315 DEBUG:teuthology.orchestra.run.smithi053:> sudo service rsyslog restart 2024-09-13T05:14:45.317 DEBUG:teuthology.orchestra.run.smithi110:> sudo service rsyslog restart 2024-09-13T05:14:45.364 INFO:teuthology.orchestra.run.smithi053.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T05:14:45.414 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T05:14:45.785 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T05:14:45.792 INFO:teuthology.task.internal:Starting timer... 2024-09-13T05:14:45.793 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T05:14:45.803 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T05:14:45.812 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-13T05:14:45.812 DEBUG:teuthology.orchestra.run.smithi053:> sudo service auditd rotate 2024-09-13T05:14:45.858 INFO:teuthology.orchestra.run.smithi053.stdout:Rotating logs: 2024-09-13T05:14:45.860 DEBUG:teuthology.orchestra.run.smithi110:> sudo service auditd rotate 2024-09-13T05:14:45.898 INFO:teuthology.orchestra.run.smithi110.stdout:Rotating logs: 2024-09-13T05:14:45.900 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T05:14:45.901 DEBUG:teuthology.orchestra.run.smithi053:> /usr/sbin/getenforce 2024-09-13T05:14:45.932 INFO:teuthology.orchestra.run.smithi053.stdout:Permissive 2024-09-13T05:14:45.932 DEBUG:teuthology.orchestra.run.smithi110:> /usr/sbin/getenforce 2024-09-13T05:14:45.966 INFO:teuthology.orchestra.run.smithi110.stdout:Permissive 2024-09-13T05:14:45.966 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi053.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi110.front.sepia.ceph.com': 'permissive'} 2024-09-13T05:14:45.966 DEBUG:teuthology.orchestra.run.smithi053:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T05:14:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:14:46.000 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T05:14:46.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:14:46.032 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T05:14:46.032 DEBUG:teuthology.orchestra.run.smithi053:> sudo /usr/sbin/setenforce permissive 2024-09-13T05:14:46.076 DEBUG:teuthology.orchestra.run.smithi110:> sudo /usr/sbin/setenforce permissive 2024-09-13T05:14:46.111 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T05:14:46.121 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-13T05:14:46.125 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-13T05:14:46.245 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T05:14:46.257 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T05:14:46.258 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi053.front.sepia.ceph.com,smithi110.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T05:18:33.711 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi053.front.sepia.ceph.com'), Remote(name='ubuntu@smithi110.front.sepia.ceph.com')] 2024-09-13T05:18:33.713 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-09-13T05:18:33.714 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi053.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:18:33.800 DEBUG:teuthology.orchestra.run.smithi053:> true 2024-09-13T05:18:33.881 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi053.front.sepia.ceph.com' 2024-09-13T05:18:33.882 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-13T05:18:33.882 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi110.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T05:18:33.965 DEBUG:teuthology.orchestra.run.smithi110:> true 2024-09-13T05:18:34.039 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi110.front.sepia.ceph.com' 2024-09-13T05:18:34.039 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T05:18:34.049 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T05:18:34.050 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T05:18:34.050 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T05:18:34.052 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T05:18:34.053 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T05:18:34.083 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T05:18:34.100 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T05:18:34.108 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T05:18:34.122 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T05:18:34.126 INFO:teuthology.orchestra.run.smithi053.stderr:sudo: ntpd: command not found 2024-09-13T05:18:34.138 INFO:teuthology.orchestra.run.smithi053.stdout:506 Cannot talk to daemon 2024-09-13T05:18:34.145 INFO:teuthology.orchestra.run.smithi110.stderr:sudo: ntpd: command not found 2024-09-13T05:18:34.154 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T05:18:34.156 INFO:teuthology.orchestra.run.smithi110.stdout:506 Cannot talk to daemon 2024-09-13T05:18:34.169 INFO:teuthology.orchestra.run.smithi053.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T05:18:34.170 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T05:18:34.184 INFO:teuthology.orchestra.run.smithi110.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T05:18:34.228 INFO:teuthology.orchestra.run.smithi053.stderr:bash: line 1: ntpq: command not found 2024-09-13T05:18:34.231 INFO:teuthology.orchestra.run.smithi053.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T05:18:34.231 INFO:teuthology.orchestra.run.smithi053.stdout:=============================================================================== 2024-09-13T05:18:34.231 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.231 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.231 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.231 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.242 INFO:teuthology.orchestra.run.smithi110.stderr:bash: line 1: ntpq: command not found 2024-09-13T05:18:34.246 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T05:18:34.246 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2024-09-13T05:18:34.246 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.246 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.246 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.246 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:18:34.247 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T05:18:34.342 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3'} 2024-09-13T05:18:34.342 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-13T05:18:34.343 INFO:tasks.cephadm:Cluster fsid is 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:18:34.343 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T05:18:34.343 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-13T05:18:34.343 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi053': '172.21.15.53', 'mon.smithi110': '172.21.15.110'} 2024-09-13T05:18:34.343 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T05:18:34.343 DEBUG:teuthology.orchestra.run.smithi053:> sudo hostname $(hostname -s) 2024-09-13T05:18:34.370 DEBUG:teuthology.orchestra.run.smithi110:> sudo hostname $(hostname -s) 2024-09-13T05:18:34.397 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-13T05:18:34.397 DEBUG:teuthology.orchestra.run.smithi053:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T05:18:34.716 INFO:teuthology.orchestra.run.smithi053.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 05:18 /home/ubuntu/cephtest/cephadm 2024-09-13T05:18:34.717 DEBUG:teuthology.orchestra.run.smithi110:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T05:18:34.843 INFO:teuthology.orchestra.run.smithi110.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 05:18 /home/ubuntu/cephtest/cephadm 2024-09-13T05:18:34.843 DEBUG:teuthology.orchestra.run.smithi053:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T05:18:34.861 DEBUG:teuthology.orchestra.run.smithi110:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T05:18:34.888 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-13T05:18:34.888 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T05:18:34.902 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T05:18:36.481 INFO:teuthology.orchestra.run.smithi110.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T05:18:36.531 INFO:teuthology.orchestra.run.smithi053.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T05:18:48.501 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:18:48.501 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T05:18:48.502 INFO:teuthology.orchestra.run.smithi053.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T05:18:48.502 INFO:teuthology.orchestra.run.smithi053.stdout: "repo_digests": [ 2024-09-13T05:18:48.502 INFO:teuthology.orchestra.run.smithi053.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T05:18:48.502 INFO:teuthology.orchestra.run.smithi053.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T05:18:48.502 INFO:teuthology.orchestra.run.smithi053.stdout: ] 2024-09-13T05:18:48.502 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout:{ 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout: "repo_digests": [ 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout: ] 2024-09-13T05:18:48.668 INFO:teuthology.orchestra.run.smithi110.stdout:} 2024-09-13T05:18:48.686 DEBUG:teuthology.orchestra.run.smithi053:> sudo mkdir -p /etc/ceph 2024-09-13T05:18:48.715 DEBUG:teuthology.orchestra.run.smithi110:> sudo mkdir -p /etc/ceph 2024-09-13T05:18:48.744 DEBUG:teuthology.orchestra.run.smithi053:> sudo chmod 777 /etc/ceph 2024-09-13T05:18:48.782 DEBUG:teuthology.orchestra.run.smithi110:> sudo chmod 777 /etc/ceph 2024-09-13T05:18:48.808 INFO:tasks.cephadm:Writing seed config... 2024-09-13T05:18:48.809 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T05:18:48.810 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T05:18:48.810 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T05:18:48.810 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T05:18:48.810 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T05:18:48.810 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T05:18:48.810 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T05:18:48.810 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T05:18:48.811 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:18:48.811 DEBUG:teuthology.orchestra.run.smithi053:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T05:18:48.839 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9f95c28e-718f-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T05:18:48.839 DEBUG:teuthology.orchestra.run.smithi053:mon.smithi053> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service 2024-09-13T05:18:48.881 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T05:18:48.881 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.53 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T05:18:49.079 INFO:teuthology.orchestra.run.smithi053.stderr:-------------------------------------------------------------------------------- 2024-09-13T05:18:49.079 INFO:teuthology.orchestra.run.smithi053.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '9f95c28e-718f-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.53', '--skip-admin-label'] 2024-09-13T05:18:49.101 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: 5.2.2 2024-09-13T05:18:49.104 INFO:teuthology.orchestra.run.smithi053.stderr:Verifying podman|docker is present... 2024-09-13T05:18:49.124 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: 5.2.2 2024-09-13T05:18:49.126 INFO:teuthology.orchestra.run.smithi053.stderr:Verifying lvm2 is present... 2024-09-13T05:18:49.126 INFO:teuthology.orchestra.run.smithi053.stderr:Verifying time synchronization is in place... 2024-09-13T05:18:49.134 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T05:18:49.140 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: inactive 2024-09-13T05:18:49.147 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: enabled 2024-09-13T05:18:49.154 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: active 2024-09-13T05:18:49.155 INFO:teuthology.orchestra.run.smithi053.stderr:Unit chronyd.service is enabled and running 2024-09-13T05:18:49.155 INFO:teuthology.orchestra.run.smithi053.stderr:Repeating the final host check... 2024-09-13T05:18:49.174 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: 5.2.2 2024-09-13T05:18:49.177 INFO:teuthology.orchestra.run.smithi053.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-13T05:18:49.177 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl is present 2024-09-13T05:18:49.177 INFO:teuthology.orchestra.run.smithi053.stderr:lvcreate is present 2024-09-13T05:18:49.183 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T05:18:49.190 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: inactive 2024-09-13T05:18:49.197 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: enabled 2024-09-13T05:18:49.203 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: active 2024-09-13T05:18:49.204 INFO:teuthology.orchestra.run.smithi053.stderr:Unit chronyd.service is enabled and running 2024-09-13T05:18:49.204 INFO:teuthology.orchestra.run.smithi053.stderr:Host looks OK 2024-09-13T05:18:49.204 INFO:teuthology.orchestra.run.smithi053.stderr:Cluster fsid: 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:18:49.204 INFO:teuthology.orchestra.run.smithi053.stderr:Acquiring lock 140450332948368 on /run/cephadm/9f95c28e-718f-11ef-bceb-c7b262605968.lock 2024-09-13T05:18:49.204 INFO:teuthology.orchestra.run.smithi053.stderr:Lock 140450332948368 acquired on /run/cephadm/9f95c28e-718f-11ef-bceb-c7b262605968.lock 2024-09-13T05:18:49.205 INFO:teuthology.orchestra.run.smithi053.stderr:Verifying IP 172.21.15.53 port 3300 ... 2024-09-13T05:18:49.205 INFO:teuthology.orchestra.run.smithi053.stderr:Verifying IP 172.21.15.53 port 6789 ... 2024-09-13T05:18:49.205 INFO:teuthology.orchestra.run.smithi053.stderr:Base mon IP is 172.21.15.53, final addrv is [v2:172.21.15.53:3300,v1:172.21.15.53:6789] 2024-09-13T05:18:49.207 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.53 metric 100 2024-09-13T05:18:49.208 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.53 metric 100 2024-09-13T05:18:49.210 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T05:18:49.210 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-13T05:18:49.211 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1792sec hoplimit 64 pref medium 2024-09-13T05:18:49.213 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T05:18:49.213 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-13T05:18:49.213 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T05:18:49.213 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-13T05:18:49.214 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fef8:3718/64 scope link 2024-09-13T05:18:49.214 INFO:teuthology.orchestra.run.smithi053.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T05:18:49.214 INFO:teuthology.orchestra.run.smithi053.stderr:Mon IP `172.21.15.53` is in CIDR network `172.21.0.0/20` 2024-09-13T05:18:49.214 INFO:teuthology.orchestra.run.smithi053.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T05:18:49.215 INFO:teuthology.orchestra.run.smithi053.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T05:18:49.317 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-13T05:18:50.333 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Getting image source signatures 2024-09-13T05:18:50.333 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-13T05:18:50.334 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-13T05:18:50.334 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-13T05:18:50.334 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-13T05:18:50.334 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-13T05:18:50.334 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T05:18:50.497 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: Writing manifest to image destination 2024-09-13T05:18:50.641 INFO:teuthology.orchestra.run.smithi053.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T05:18:51.070 INFO:teuthology.orchestra.run.smithi053.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T05:18:51.555 INFO:teuthology.orchestra.run.smithi053.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T05:18:51.555 INFO:teuthology.orchestra.run.smithi053.stderr:Extracting ceph user uid/gid from container image... 2024-09-13T05:18:51.964 INFO:teuthology.orchestra.run.smithi053.stderr:stat: 167 167 2024-09-13T05:18:52.378 INFO:teuthology.orchestra.run.smithi053.stderr:Creating initial keys... 2024-09-13T05:18:54.327 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph-authtool: AQA9y+Nml/hAIhAAqeuVRIlIZ59iZ6ot6yBh2g== 2024-09-13T05:19:03.858 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph-authtool: AQBHy+NmBAFYBBAAnvUNkaG98dAQBGijHQ1Ycg== 2024-09-13T05:19:05.254 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph-authtool: AQBJy+NmMHtDChAAFFrE9to2DPt0q39JhJbRBA== 2024-09-13T05:19:05.996 INFO:teuthology.orchestra.run.smithi053.stderr:Creating initial monmap... 2024-09-13T05:19:06.344 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T05:19:06.344 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-13T05:19:06.344 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:06.344 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T05:19:06.834 INFO:teuthology.orchestra.run.smithi053.stderr:monmaptool for smithi053 [v2:172.21.15.53:3300,v1:172.21.15.53:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T05:19:06.834 INFO:teuthology.orchestra.run.smithi053.stderr:setting min_mon_release = octopus 2024-09-13T05:19:06.834 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/monmaptool: set fsid to 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:06.834 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T05:19:06.834 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:19:06.834 INFO:teuthology.orchestra.run.smithi053.stderr:Creating mon... 2024-09-13T05:19:08.155 INFO:teuthology.orchestra.run.smithi053.stderr:create mon.smithi053 on 2024-09-13T05:19:08.375 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T05:19:08.559 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9f95c28e-718f-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-9f95c28e-718f-11ef-bceb-c7b262605968.target. 2024-09-13T05:19:08.559 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-9f95c28e-718f-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-9f95c28e-718f-11ef-bceb-c7b262605968.target. 2024-09-13T05:19:08.869 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Failed to reset failed state of unit ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service: Unit ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service not loaded. 2024-09-13T05:19:08.877 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9f95c28e-718f-11ef-bceb-c7b262605968.target.wants/ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service → /etc/systemd/system/ceph-9f95c28e-718f-11ef-bceb-c7b262605968@.service. 2024-09-13T05:19:09.930 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: disabled 2024-09-13T05:19:09.941 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: inactive 2024-09-13T05:19:09.941 INFO:teuthology.orchestra.run.smithi053.stderr:firewalld.service is not enabled 2024-09-13T05:19:09.941 INFO:teuthology.orchestra.run.smithi053.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T05:19:09.941 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for mon to start... 2024-09-13T05:19:09.941 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for mon... 2024-09-13T05:19:10.459 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: cluster: 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: id: 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: services: 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi053 (age 0.592416s) 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: data: 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-13T05:19:10.460 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-13T05:19:10.461 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-13T05:19:10.461 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: pgs: 2024-09-13T05:19:10.461 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:11.050 INFO:teuthology.orchestra.run.smithi053.stderr:mon is available 2024-09-13T05:19:11.050 INFO:teuthology.orchestra.run.smithi053.stderr:Assimilating anything we can from ceph.conf... 2024-09-13T05:19:11.532 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:11.532 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: [global] 2024-09-13T05:19:11.532 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: fsid = 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:11.532 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.53:3300,v1:172.21.15.53:6789] 2024-09-13T05:19:11.532 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T05:19:11.532 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: [mgr] 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: [osd] 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T05:19:11.533 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T05:19:12.055 INFO:teuthology.orchestra.run.smithi053.stderr:Generating new minimal ceph.conf... 2024-09-13T05:19:13.120 INFO:teuthology.orchestra.run.smithi053.stderr:Restarting the monitor... 2024-09-13T05:19:13.139 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 systemd[1]: Stopping Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:19:13.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 bash[23598]: Error: no container with name or ID "ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon.smithi053" found: no such container 2024-09-13T05:19:13.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[23364]: 2024-09-13T05:19:13.286+0000 7f126b6a6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi053 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:19:13.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[23364]: 2024-09-13T05:19:13.286+0000 7f126b6a6700 -1 mon.smithi053@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T05:19:13.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 podman[23609]: 2024-09-13 05:19:13.307437163 +0000 UTC m=+0.133015502 container died 1ea330ed080e73cf666af5efb1daa0e50729d9a1ae8492c402507c298ab06d96 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, version=8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=centos-stream-container, vcs-type=git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, GIT_CLEAN=True, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T05:19:13.945 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 podman[23609]: 2024-09-13 05:19:13.661190063 +0000 UTC m=+0.486768392 container cleanup 1ea330ed080e73cf666af5efb1daa0e50729d9a1ae8492c402507c298ab06d96 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, RELEASE=HEAD, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , version=8, vcs-type=git, io.openshift.tags=base centos centos-stream, distribution-scope=public, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T05:19:13.945 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 bash[23609]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053 2024-09-13T05:19:13.945 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 bash[23636]: Error: no container with name or ID "ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon.smithi053" found: no such container 2024-09-13T05:19:14.196 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 podman[23621]: 2024-09-13 05:19:13.945190498 +0000 UTC m=+0.640465514 container remove 1ea330ed080e73cf666af5efb1daa0e50729d9a1ae8492c402507c298ab06d96 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., release=754, vcs-type=git, version=8, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, ceph=True, architecture=x86_64) 2024-09-13T05:19:14.196 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service: Deactivated successfully. 2024-09-13T05:19:14.196 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 systemd[1]: Stopped Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:19:14.196 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:13 smithi053 systemd[1]: Starting Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:19:14.196 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 podman[23703]: 2024-09-13 05:19:14.086011544 +0000 UTC m=+0.018320116 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T05:19:14.196 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 podman[23703]: 2024-09-13 05:19:14.195861496 +0000 UTC m=+0.128170046 container create 5d17dc93b2a5b1c4e65e307cbb786562415ef5e905001f3133b4067443c57822 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, release=754, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, version=8, RELEASE=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc.) 2024-09-13T05:19:14.456 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 podman[23703]: 2024-09-13 05:19:14.38152597 +0000 UTC m=+0.313834521 container init 5d17dc93b2a5b1c4e65e307cbb786562415ef5e905001f3133b4067443c57822 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, architecture=x86_64, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 podman[23703]: 2024-09-13 05:19:14.38474033 +0000 UTC m=+0.317048881 container start 5d17dc93b2a5b1c4e65e307cbb786562415ef5e905001f3133b4067443c57822 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, architecture=x86_64, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, version=8) 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: pidfile_write: ignore empty --pid-file 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: load: jerasure load: lrc 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: RocksDB version: 6.15.5 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Compile date Apr 18 2022 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: DB SUMMARY 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: DB Session ID: QYXAEOU07G8NVCJDCA4J 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: CURRENT file: CURRENT 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T05:19:14.457 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi053/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi053/store.db: 000010.log size: 82709 ; 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.error_if_exists: 0 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.create_if_missing: 0 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.paranoid_checks: 1 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.env: 0x5584b6cc6860 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.fs: Posix File System 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.info_log: 0x5584b8ac7fa0 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.statistics: (nil) 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.use_fsync: 0 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_log_file_size: 0 2024-09-13T05:19:14.458 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.allow_fallocate: 1 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.use_direct_reads: 0 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.db_log_dir: 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi053/store.db 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T05:19:14.459 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T05:19:14.460 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T05:19:14.460 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T05:19:14.460 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T05:19:14.460 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T05:19:14.460 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T05:19:14.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T05:19:14.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.write_buffer_manager: 0x5584b8bb8330 2024-09-13T05:19:14.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.unordered_write: 0 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T05:19:14.462 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.row_cache: None 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.wal_filter: None 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.preserve_deletes: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.two_write_queues: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.atomic_flush: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.log_readahead_size: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T05:19:14.463 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_background_jobs: 2 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_background_compactions: -1 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_subcompactions: 1 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T05:19:14.464 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_open_files: -1 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_background_flushes: -1 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Compression algorithms supported: 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kZSTD supported: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kXpressCompression supported: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kLZ4Compression supported: 1 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kBZip2Compression supported: 0 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kZlibCompression supported: 1 2024-09-13T05:19:14.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: kSnappyCompression supported: 1 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi053/store.db/MANIFEST-000009 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.merge_operator: 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_filter: None 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5584b8a93d00) 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: cache_index_and_filter_blocks: 1 2024-09-13T05:19:14.466 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: pin_top_level_index_and_filter: 1 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: index_type: 0 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: data_block_index_type: 0 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: index_shortening: 1 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: hash_index_allow_collision: 1 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: checksum: 1 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: no_block_cache: 0 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache: 0x5584b8afe170 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache_name: BinnedLRUCache 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache_options: 2024-09-13T05:19:14.467 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: capacity : 536870912 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: num_shard_bits : 4 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: strict_capacity_limit : 0 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: high_pri_pool_ratio: 0.000 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache_compressed: (nil) 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: persistent_cache: (nil) 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_size: 4096 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_size_deviation: 10 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_restart_interval: 16 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: index_block_restart_interval: 1 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: metadata_block_size: 4096 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: partition_filters: 0 2024-09-13T05:19:14.468 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: use_delta_encoding: 1 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: whole_key_filtering: 1 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: verify_compression: 0 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: read_amp_bytes_per_bit: 0 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: format_version: 4 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: enable_index_compression: 1 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_align: 0 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T05:19:14.469 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression: NoCompression 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.num_levels: 7 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T05:19:14.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T05:19:14.471 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T05:19:14.472 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.table_properties_collectors: 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.inplace_update_support: 0 2024-09-13T05:19:14.473 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.bloom_locality: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.max_successive_merges: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.ttl: 2592000 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.enable_blob_files: false 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.min_blob_size: 0 2024-09-13T05:19:14.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi053/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726204754411627, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726204754412396, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79391, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77655, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71964, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726204754, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "55e9e85d-c981-474e-8a47-c0b879431a35", "db_session_id": "QYXAEOU07G8NVCJDCA4J"}} 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726204754412737, "job": 1, "event": "recovery_finished"} 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi053/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T05:19:14.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5584b8ae4a80 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: DB pointer 0x5584b8af4000 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** DB Stats ** 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:19:14.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** Compaction Stats [default] ** 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** Compaction Stats [default] ** 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T05:19:14.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative compaction: 0.00 GB write, 24.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval compaction: 0.00 GB write, 24.07 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** Compaction Stats [default] ** 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:19:14.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** Compaction Stats [default] ** 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative compaction: 0.00 GB write, 23.83 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:19:14.479 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: starting mon.smithi053 rank 0 at public addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] at bind addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi053 fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???) e1 preinit fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???).mds e1 new map 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???).mds e1 print_map 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: e1 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: legacy client fscid: -1 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: No filesystems configured 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T05:19:14.480 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expand map: {default=false} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta from 'false' to 'false' 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expanded map: {default=false} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expand map: {default=info} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta from 'info' to 'info' 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expanded map: {default=info} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expand map: {default=daemon} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta from 'daemon' to 'daemon' 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expanded map: {default=daemon} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expand map: {default=debug} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta from 'debug' to 'debug' 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: expand_channel_meta expanded map: {default=debug} 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 bash[23703]: 5d17dc93b2a5b1c4e65e307cbb786562415ef5e905001f3133b4067443c57822 2024-09-13T05:19:14.481 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 systemd[1]: Started Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:19:14.482 INFO:teuthology.orchestra.run.smithi053.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-13T05:19:14.713 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mon.smithi053 is new leader, mons smithi053 in quorum (ranks 0) 2024-09-13T05:19:14.713 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: monmap e1: 1 mons at {smithi053=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0]} 2024-09-13T05:19:14.713 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: fsmap 2024-09-13T05:19:14.713 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T05:19:14.713 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:14 smithi053 ceph-mon[23721]: mgrmap e1: no daemons active 2024-09-13T05:19:15.608 INFO:teuthology.orchestra.run.smithi053.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-13T05:19:15.608 INFO:teuthology.orchestra.run.smithi053.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T05:19:15.608 INFO:teuthology.orchestra.run.smithi053.stderr:Creating mgr... 2024-09-13T05:19:15.608 INFO:teuthology.orchestra.run.smithi053.stderr:Verifying port 9283 ... 2024-09-13T05:19:15.800 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Failed to reset failed state of unit ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mgr.smithi053.ygzjut.service: Unit ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mgr.smithi053.ygzjut.service not loaded. 2024-09-13T05:19:15.808 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9f95c28e-718f-11ef-bceb-c7b262605968.target.wants/ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mgr.smithi053.ygzjut.service → /etc/systemd/system/ceph-9f95c28e-718f-11ef-bceb-c7b262605968@.service. 2024-09-13T05:19:16.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:16 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3916520879' entity='client.admin' 2024-09-13T05:19:16.762 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: disabled 2024-09-13T05:19:16.770 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: inactive 2024-09-13T05:19:16.770 INFO:teuthology.orchestra.run.smithi053.stderr:firewalld.service is not enabled 2024-09-13T05:19:16.771 INFO:teuthology.orchestra.run.smithi053.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T05:19:16.779 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: disabled 2024-09-13T05:19:16.786 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: inactive 2024-09-13T05:19:16.788 INFO:teuthology.orchestra.run.smithi053.stderr:firewalld.service is not enabled 2024-09-13T05:19:16.788 INFO:teuthology.orchestra.run.smithi053.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-13T05:19:16.788 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for mgr to start... 2024-09-13T05:19:16.788 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for mgr... 2024-09-13T05:19:17.386 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsid": "9f95c28e-718f-11ef-bceb-c7b262605968", 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "health": { 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 0 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T05:19:17.387 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "smithi053" 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T05:19:17.388 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T05:19:17.392 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "available": false, 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "iostat", 2024-09-13T05:19:17.393 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "nfs", 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "restful" 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modified": "2024-09-13T05:19:09.865469+0000", 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T05:19:17.394 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:19:17.688 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:17 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/348888326' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T05:19:17.992 INFO:teuthology.orchestra.run.smithi053.stderr:mgr not available, waiting (1/15)... 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsid": "9f95c28e-718f-11ef-bceb-c7b262605968", 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "health": { 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T05:19:20.660 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 0 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "smithi053" 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_age": 6, 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:20.661 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T05:19:20.662 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:20.662 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T05:19:20.662 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T05:19:20.662 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T05:19:20.662 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T05:19:20.665 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T05:19:20.665 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T05:19:20.665 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:20.665 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T05:19:20.665 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T05:19:20.665 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T05:19:20.665 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "available": false, 2024-09-13T05:19:20.666 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "iostat", 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "nfs", 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "restful" 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modified": "2024-09-13T05:19:09.865469+0000", 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T05:19:20.667 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:19:20.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:20 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3908221430' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T05:19:21.281 INFO:teuthology.orchestra.run.smithi053.stderr:mgr not available, waiting (2/15)... 2024-09-13T05:19:23.404 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: Activating manager daemon smithi053.ygzjut 2024-09-13T05:19:23.404 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: mgrmap e2: smithi053.ygzjut(active, starting, since 0.00303041s) 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:19:23.405 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:23.406 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:23.406 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:23 smithi053 ceph-mon[23721]: from='mgr.14100 172.21.15.53:0/1848560941' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:23.776 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:23.776 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:19:23.776 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsid": "9f95c28e-718f-11ef-bceb-c7b262605968", 2024-09-13T05:19:23.776 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "health": { 2024-09-13T05:19:23.776 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T05:19:23.776 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T05:19:23.776 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 0 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "smithi053" 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T05:19:23.777 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T05:19:23.778 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "available": false, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T05:19:23.779 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "iostat", 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "nfs", 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "restful" 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modified": "2024-09-13T05:19:09.865469+0000", 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T05:19:23.780 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:19:24.359 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:24 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3109078542' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T05:19:24.360 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:24 smithi053 ceph-mon[23721]: mgrmap e3: smithi053.ygzjut(active, since 1.00568s) 2024-09-13T05:19:24.362 INFO:teuthology.orchestra.run.smithi053.stderr:mgr not available, waiting (3/15)... 2024-09-13T05:19:26.713 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:26 smithi053 ceph-mon[23721]: mgrmap e4: smithi053.ygzjut(active, since 2s) 2024-09-13T05:19:27.075 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsid": "9f95c28e-718f-11ef-bceb-c7b262605968", 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "health": { 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 0 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:27.076 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "smithi053" 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T05:19:27.077 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T05:19:27.081 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "available": true, 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "iostat", 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "nfs", 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "restful" 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ], 2024-09-13T05:19:27.082 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "modified": "2024-09-13T05:19:09.865469+0000", 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "services": {} 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: }, 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T05:19:27.083 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:19:27.660 INFO:teuthology.orchestra.run.smithi053.stderr:mgr is available 2024-09-13T05:19:27.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:27 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/214472188' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T05:19:28.247 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: [global] 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: fsid = 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: [mgr] 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: [osd] 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T05:19:28.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T05:19:28.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:28 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3295731629' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T05:19:28.808 INFO:teuthology.orchestra.run.smithi053.stderr:Enabling cephadm module... 2024-09-13T05:19:30.675 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:30 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/239638865' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T05:19:31.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:31 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/239638865' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T05:19:31.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:31 smithi053 ceph-mon[23721]: mgrmap e5: smithi053.ygzjut(active, since 7s) 2024-09-13T05:19:31.670 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:19:31.670 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-13T05:19:31.670 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "available": true, 2024-09-13T05:19:31.670 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "active_name": "smithi053.ygzjut", 2024-09-13T05:19:31.670 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T05:19:31.670 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:19:32.319 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for the mgr to restart... 2024-09-13T05:19:32.320 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for mgr epoch 5... 2024-09-13T05:19:32.627 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:32 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/8199460' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T05:19:37.828 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:19:37.828 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-13T05:19:37.828 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T05:19:37.828 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: Active manager daemon smithi053.ygzjut restarted 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: Activating manager daemon smithi053.ygzjut 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: mgrmap e6: smithi053.ygzjut(active, starting, since 0.0522695s) 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:19:38.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:19:38.132 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:38.132 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:38.132 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:19:38.132 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:19:38.132 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:19:38.132 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:19:38.132 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:37 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:19:38.458 INFO:teuthology.orchestra.run.smithi053.stderr:mgr epoch 5 is available 2024-09-13T05:19:38.458 INFO:teuthology.orchestra.run.smithi053.stderr:Setting orchestrator backend to cephadm... 2024-09-13T05:19:39.081 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:38 smithi053 ceph-mon[23721]: mgrmap e7: smithi053.ygzjut(active, since 1.05488s) 2024-09-13T05:19:39.081 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:38 smithi053 ceph-mon[23721]: [13/Sep/2024:05:19:38] ENGINE Bus STARTING 2024-09-13T05:19:39.081 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:38 smithi053 ceph-mon[23721]: [13/Sep/2024:05:19:38] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:19:39.082 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:38 smithi053 ceph-mon[23721]: [13/Sep/2024:05:19:38] ENGINE Bus STARTED 2024-09-13T05:19:39.082 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:38 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:39.082 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:38 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:19:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:40 smithi053 ceph-mon[23721]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:40 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:40.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:40 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:19:40.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:40 smithi053 ceph-mon[23721]: mgrmap e8: smithi053.ygzjut(active, since 2s) 2024-09-13T05:19:40.248 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: value unchanged 2024-09-13T05:19:40.879 INFO:teuthology.orchestra.run.smithi053.stderr:Generating ssh key... 2024-09-13T05:19:41.186 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:41 smithi053 ceph-mon[23721]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:42.935 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:42 smithi053 ceph-mon[23721]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:42.936 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:42 smithi053 ceph-mon[23721]: Generating ssh key... 2024-09-13T05:19:42.936 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:42 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:42.936 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:42 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:42.936 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNaoFcTauydNG66h4BSd0UYNjmgS5UGZfsrHW4Wlc7xR17vf6Ow/DyA9iq1c+e6yQFr0Oz1XxvB6eq6KZS3Tw/pYOUK4KgMb/RC1HfTfFihWyb3PCSyYvvu+1c0/CD/03AmbxgaBKpZ3Ej6zFHsUpQZ0HqnIHkuOpchVnWrsEtf7dUNS85gbgNctxjcpq5tpFv5v3LaMsnG5m+tT+MaGLU8mzBts3mXbOzip72xQAhJgNMAtKYKdoeCSvwyJR1fO3IE3pkJjzYI+DmcCEHQsoN4ZRBQSdTVGYdlZHuNJCLUtKw4XPBlKcX3pcpneQGZLtDn9vIZqVga1HJPnLJNZ4ZxjiCZPhMy+saaO1xuwxZdbJjMjOY+Y2yiwxkgkewwOYeqzHIT2J6992EhYHysnNYqGGV1qvGfxf31v0mHh9jAo7G0kHVritUp1wb4+oDmJGQKNO8d2C6vCM5J9QTnQ2aWPE/HKniFdIkmf1i/zgap6Cdz1IDZVH8vZ06IOGeqD8= ceph-9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:19:43.500 INFO:teuthology.orchestra.run.smithi053.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T05:19:43.500 INFO:teuthology.orchestra.run.smithi053.stderr:Adding key to root@localhost authorized_keys... 2024-09-13T05:19:43.505 INFO:teuthology.orchestra.run.smithi053.stderr:Adding host smithi053... 2024-09-13T05:19:43.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:43 smithi053 ceph-mon[23721]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:45.061 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Added host 'smithi053' with addr '172.21.15.53' 2024-09-13T05:19:45.606 INFO:teuthology.orchestra.run.smithi053.stderr:Deploying mon service with default placement... 2024-09-13T05:19:45.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:45 smithi053 ceph-mon[23721]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi053", "addr": "172.21.15.53", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:45.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:45 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:45.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:45 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:19:46.195 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-13T05:19:46.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:46 smithi053 ceph-mon[23721]: Deploying cephadm binary to smithi053 2024-09-13T05:19:46.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:46 smithi053 ceph-mon[23721]: Added host smithi053 2024-09-13T05:19:46.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:46 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:46.864 INFO:teuthology.orchestra.run.smithi053.stderr:Deploying mgr service with default placement... 2024-09-13T05:19:47.449 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:47 smithi053 ceph-mon[23721]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:47.449 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:47 smithi053 ceph-mon[23721]: Saving service mon spec with placement count:5 2024-09-13T05:19:47.490 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-13T05:19:48.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:48 smithi053 ceph-mon[23721]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:48.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:48 smithi053 ceph-mon[23721]: Saving service mgr spec with placement count:2 2024-09-13T05:19:48.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:48 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:48.877 INFO:teuthology.orchestra.run.smithi053.stderr:Deploying crash service with default placement... 2024-09-13T05:19:49.582 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-13T05:19:49.810 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:49 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:49.810 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:49 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:50.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:50 smithi053 ceph-mon[23721]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:50.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:50 smithi053 ceph-mon[23721]: Saving service crash spec with placement * 2024-09-13T05:19:50.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:50 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:50.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:50 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:50.820 INFO:teuthology.orchestra.run.smithi053.stderr:Deploying prometheus service with default placement... 2024-09-13T05:19:51.453 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-13T05:19:52.043 INFO:teuthology.orchestra.run.smithi053.stderr:Deploying grafana service with default placement... 2024-09-13T05:19:52.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:52 smithi053 ceph-mon[23721]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:52.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:52 smithi053 ceph-mon[23721]: Saving service prometheus spec with placement count:1 2024-09-13T05:19:52.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:52 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:52.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:52 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:52.704 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-13T05:19:53.695 INFO:teuthology.orchestra.run.smithi053.stderr:Deploying node-exporter service with default placement... 2024-09-13T05:19:53.946 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:53 smithi053 ceph-mon[23721]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:53.946 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:53 smithi053 ceph-mon[23721]: Saving service grafana spec with placement count:1 2024-09-13T05:19:53.946 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:53 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:54.828 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-13T05:19:55.437 INFO:teuthology.orchestra.run.smithi053.stderr:Deploying alertmanager service with default placement... 2024-09-13T05:19:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:55 smithi053 ceph-mon[23721]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:55 smithi053 ceph-mon[23721]: Saving service node-exporter spec with placement * 2024-09-13T05:19:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:55 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:56.094 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-13T05:19:57.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:57 smithi053 ceph-mon[23721]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:19:57.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:57 smithi053 ceph-mon[23721]: Saving service alertmanager spec with placement count:1 2024-09-13T05:19:57.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:57 smithi053 ceph-mon[23721]: from='mgr.14120 172.21.15.53:0/212337566' entity='mgr.smithi053.ygzjut' 2024-09-13T05:19:58.670 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:58 smithi053 ceph-mon[23721]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:19:58.670 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:58 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3757051544' entity='client.admin' 2024-09-13T05:19:59.258 INFO:teuthology.orchestra.run.smithi053.stderr:Enabling the dashboard module... 2024-09-13T05:19:59.873 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:59 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2874402955' entity='client.admin' 2024-09-13T05:19:59.873 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:19:59 smithi053 ceph-mon[23721]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:00.918 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:00 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3818827097' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T05:20:01.732 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:01 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3818827097' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T05:20:01.732 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:01 smithi053 ceph-mon[23721]: mgrmap e9: smithi053.ygzjut(active, since 23s) 2024-09-13T05:20:02.072 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:20:02.072 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-13T05:20:02.072 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "available": true, 2024-09-13T05:20:02.073 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "active_name": "smithi053.ygzjut", 2024-09-13T05:20:02.073 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T05:20:02.073 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:20:02.728 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:02 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/188266931' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T05:20:02.731 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for the mgr to restart... 2024-09-13T05:20:02.731 INFO:teuthology.orchestra.run.smithi053.stderr:Waiting for mgr epoch 9... 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: Active manager daemon smithi053.ygzjut restarted 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: Activating manager daemon smithi053.ygzjut 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: mgrmap e10: smithi053.ygzjut(active, starting, since 0.0528069s) 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:20:08.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:20:08.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:20:08.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:20:08.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:20:08.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:20:08.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:08 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:20:08.121 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: { 2024-09-13T05:20:08.122 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-13T05:20:08.122 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T05:20:08.122 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: } 2024-09-13T05:20:08.884 INFO:teuthology.orchestra.run.smithi053.stderr:mgr epoch 9 is available 2024-09-13T05:20:08.884 INFO:teuthology.orchestra.run.smithi053.stderr:Generating a dashboard self-signed certificate... 2024-09-13T05:20:09.134 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:09 smithi053 ceph-mon[23721]: mgrmap e11: smithi053.ygzjut(active, since 1.05564s) 2024-09-13T05:20:09.134 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:09 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:09.134 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:09 smithi053 ceph-mon[23721]: [13/Sep/2024:05:20:08] ENGINE Bus STARTING 2024-09-13T05:20:09.134 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:09 smithi053 ceph-mon[23721]: [13/Sep/2024:05:20:08] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:20:09.134 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:09 smithi053 ceph-mon[23721]: [13/Sep/2024:05:20:08] ENGINE Bus STARTED 2024-09-13T05:20:09.134 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:09 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:09.658 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-13T05:20:10.356 INFO:teuthology.orchestra.run.smithi053.stderr:Creating initial admin user... 2024-09-13T05:20:10.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:10 smithi053 ceph-mon[23721]: mgrmap e12: smithi053.ygzjut(active, since 2s) 2024-09-13T05:20:10.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:10 smithi053 ceph-mon[23721]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:20:10.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:10 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:10.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:10 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:10.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:10 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:11.303 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$AE0FDEZBwgetlxtA9DO1uuM1skwYEu/1A9WQQFn0Y17n9iAb69POK", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726204811, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T05:20:12.160 INFO:teuthology.orchestra.run.smithi053.stderr:Fetching dashboard port number... 2024-09-13T05:20:12.325 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:12 smithi053 ceph-mon[23721]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:20:12.325 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:12 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:12.825 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: 8443 2024-09-13T05:20:13.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:13 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3761367374' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T05:20:13.923 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: disabled 2024-09-13T05:20:13.932 INFO:teuthology.orchestra.run.smithi053.stderr:systemctl: inactive 2024-09-13T05:20:13.933 INFO:teuthology.orchestra.run.smithi053.stderr:firewalld.service is not enabled 2024-09-13T05:20:13.933 INFO:teuthology.orchestra.run.smithi053.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T05:20:13.935 INFO:teuthology.orchestra.run.smithi053.stderr:Ceph Dashboard is now available at: 2024-09-13T05:20:13.936 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:13.936 INFO:teuthology.orchestra.run.smithi053.stderr: URL: https://smithi053.front.sepia.ceph.com:8443/ 2024-09-13T05:20:13.936 INFO:teuthology.orchestra.run.smithi053.stderr: User: admin 2024-09-13T05:20:13.936 INFO:teuthology.orchestra.run.smithi053.stderr: Password: 4wq3gnkkzf 2024-09-13T05:20:13.936 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:13.936 INFO:teuthology.orchestra.run.smithi053.stderr:Enabling autotune for osd_memory_target 2024-09-13T05:20:14.446 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:14 smithi053 ceph-mon[23721]: mgrmap e13: smithi053.ygzjut(active, since 6s) 2024-09-13T05:20:16.067 INFO:teuthology.orchestra.run.smithi053.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-13T05:20:16.760 INFO:teuthology.orchestra.run.smithi053.stderr:You can access the Ceph CLI with: 2024-09-13T05:20:16.760 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:16.760 INFO:teuthology.orchestra.run.smithi053.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr: ceph telemetry on 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr:For more information see: 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:20:16.761 INFO:teuthology.orchestra.run.smithi053.stderr:Bootstrap complete. 2024-09-13T05:20:16.802 INFO:tasks.cephadm:Fetching config... 2024-09-13T05:20:16.803 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:20:16.803 DEBUG:teuthology.orchestra.run.smithi053:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T05:20:16.823 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T05:20:16.823 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:20:16.823 DEBUG:teuthology.orchestra.run.smithi053:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T05:20:16.881 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T05:20:16.882 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:20:16.882 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/keyring of=/dev/stdout 2024-09-13T05:20:16.950 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T05:20:16.950 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:20:16.950 DEBUG:teuthology.orchestra.run.smithi053:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T05:20:17.008 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T05:20:17.009 DEBUG:teuthology.orchestra.run.smithi053:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNaoFcTauydNG66h4BSd0UYNjmgS5UGZfsrHW4Wlc7xR17vf6Ow/DyA9iq1c+e6yQFr0Oz1XxvB6eq6KZS3Tw/pYOUK4KgMb/RC1HfTfFihWyb3PCSyYvvu+1c0/CD/03AmbxgaBKpZ3Ej6zFHsUpQZ0HqnIHkuOpchVnWrsEtf7dUNS85gbgNctxjcpq5tpFv5v3LaMsnG5m+tT+MaGLU8mzBts3mXbOzip72xQAhJgNMAtKYKdoeCSvwyJR1fO3IE3pkJjzYI+DmcCEHQsoN4ZRBQSdTVGYdlZHuNJCLUtKw4XPBlKcX3pcpneQGZLtDn9vIZqVga1HJPnLJNZ4ZxjiCZPhMy+saaO1xuwxZdbJjMjOY+Y2yiwxkgkewwOYeqzHIT2J6992EhYHysnNYqGGV1qvGfxf31v0mHh9jAo7G0kHVritUp1wb4+oDmJGQKNO8d2C6vCM5J9QTnQ2aWPE/HKniFdIkmf1i/zgap6Cdz1IDZVH8vZ06IOGeqD8= ceph-9f95c28e-718f-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T05:20:17.088 INFO:teuthology.orchestra.run.smithi053.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNaoFcTauydNG66h4BSd0UYNjmgS5UGZfsrHW4Wlc7xR17vf6Ow/DyA9iq1c+e6yQFr0Oz1XxvB6eq6KZS3Tw/pYOUK4KgMb/RC1HfTfFihWyb3PCSyYvvu+1c0/CD/03AmbxgaBKpZ3Ej6zFHsUpQZ0HqnIHkuOpchVnWrsEtf7dUNS85gbgNctxjcpq5tpFv5v3LaMsnG5m+tT+MaGLU8mzBts3mXbOzip72xQAhJgNMAtKYKdoeCSvwyJR1fO3IE3pkJjzYI+DmcCEHQsoN4ZRBQSdTVGYdlZHuNJCLUtKw4XPBlKcX3pcpneQGZLtDn9vIZqVga1HJPnLJNZ4ZxjiCZPhMy+saaO1xuwxZdbJjMjOY+Y2yiwxkgkewwOYeqzHIT2J6992EhYHysnNYqGGV1qvGfxf31v0mHh9jAo7G0kHVritUp1wb4+oDmJGQKNO8d2C6vCM5J9QTnQ2aWPE/HKniFdIkmf1i/zgap6Cdz1IDZVH8vZ06IOGeqD8= ceph-9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:20:17.100 DEBUG:teuthology.orchestra.run.smithi110:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNaoFcTauydNG66h4BSd0UYNjmgS5UGZfsrHW4Wlc7xR17vf6Ow/DyA9iq1c+e6yQFr0Oz1XxvB6eq6KZS3Tw/pYOUK4KgMb/RC1HfTfFihWyb3PCSyYvvu+1c0/CD/03AmbxgaBKpZ3Ej6zFHsUpQZ0HqnIHkuOpchVnWrsEtf7dUNS85gbgNctxjcpq5tpFv5v3LaMsnG5m+tT+MaGLU8mzBts3mXbOzip72xQAhJgNMAtKYKdoeCSvwyJR1fO3IE3pkJjzYI+DmcCEHQsoN4ZRBQSdTVGYdlZHuNJCLUtKw4XPBlKcX3pcpneQGZLtDn9vIZqVga1HJPnLJNZ4ZxjiCZPhMy+saaO1xuwxZdbJjMjOY+Y2yiwxkgkewwOYeqzHIT2J6992EhYHysnNYqGGV1qvGfxf31v0mHh9jAo7G0kHVritUp1wb4+oDmJGQKNO8d2C6vCM5J9QTnQ2aWPE/HKniFdIkmf1i/zgap6Cdz1IDZVH8vZ06IOGeqD8= ceph-9f95c28e-718f-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T05:20:17.168 INFO:teuthology.orchestra.run.smithi110.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDNaoFcTauydNG66h4BSd0UYNjmgS5UGZfsrHW4Wlc7xR17vf6Ow/DyA9iq1c+e6yQFr0Oz1XxvB6eq6KZS3Tw/pYOUK4KgMb/RC1HfTfFihWyb3PCSyYvvu+1c0/CD/03AmbxgaBKpZ3Ej6zFHsUpQZ0HqnIHkuOpchVnWrsEtf7dUNS85gbgNctxjcpq5tpFv5v3LaMsnG5m+tT+MaGLU8mzBts3mXbOzip72xQAhJgNMAtKYKdoeCSvwyJR1fO3IE3pkJjzYI+DmcCEHQsoN4ZRBQSdTVGYdlZHuNJCLUtKw4XPBlKcX3pcpneQGZLtDn9vIZqVga1HJPnLJNZ4ZxjiCZPhMy+saaO1xuwxZdbJjMjOY+Y2yiwxkgkewwOYeqzHIT2J6992EhYHysnNYqGGV1qvGfxf31v0mHh9jAo7G0kHVritUp1wb4+oDmJGQKNO8d2C6vCM5J9QTnQ2aWPE/HKniFdIkmf1i/zgap6Cdz1IDZVH8vZ06IOGeqD8= ceph-9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:20:17.181 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T05:20:17.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:17 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/632824132' entity='client.admin' 2024-09-13T05:20:20.739 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T05:20:20.739 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T05:20:21.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:21 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/13072829' entity='client.admin' 2024-09-13T05:20:22.658 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:22 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:22.658 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:22 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd/host:smithi053", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:20:22.659 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:22 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:22.659 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:22 smithi053 ceph-mon[23721]: Deploying daemon alertmanager.smithi053 on smithi053 2024-09-13T05:20:24.088 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi110 2024-09-13T05:20:24.088 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-13T05:20:24.088 DEBUG:teuthology.orchestra.run.smithi110:> dd of=/etc/ceph/ceph.conf 2024-09-13T05:20:24.108 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-13T05:20:24.108 DEBUG:teuthology.orchestra.run.smithi110:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:20:24.166 INFO:tasks.cephadm:Adding host smithi110 to orchestrator... 2024-09-13T05:20:24.166 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch host add smithi110 2024-09-13T05:20:24.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:24 smithi053 ceph-mon[23721]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:20:24.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:24 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:28 smithi053 ceph-mon[23721]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:28 smithi053 ceph-mon[23721]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi110", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:20:28.508 INFO:teuthology.orchestra.run.smithi053.stdout:Added host 'smithi110' with addr '172.21.15.110' 2024-09-13T05:20:29.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:29 smithi053 ceph-mon[23721]: Deploying cephadm binary to smithi110 2024-09-13T05:20:29.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:29 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:29.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:29 smithi053 ceph-mon[23721]: Added host smithi110 2024-09-13T05:20:30.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:30 smithi053 ceph-mon[23721]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:30.974 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T05:20:32.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:32 smithi053 ceph-mon[23721]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:32.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:32 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:32.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:32 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:20:32.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:32 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T05:20:32.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:32 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:20:32.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:32 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:33.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:33 smithi053 ceph-mon[23721]: Deploying daemon crash.smithi053 on smithi053 2024-09-13T05:20:34.105 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:20:34.105 INFO:teuthology.orchestra.run.smithi053.stdout:[{"addr": "172.21.15.53", "hostname": "smithi053", "labels": [], "status": ""}, {"addr": "172.21.15.110", "hostname": "smithi110", "labels": [], "status": ""}] 2024-09-13T05:20:34.861 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:34 smithi053 ceph-mon[23721]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:35.187 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T05:20:35.187 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T05:20:35.690 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:35 smithi053 ceph-mon[23721]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:20:36.774 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:36 smithi053 ceph-mon[23721]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:36.774 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:36 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:36.774 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:36 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:36.774 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:36 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:36.774 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:36 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:20:36.774 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:36 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:37.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:37 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:20:37.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:37 smithi053 ceph-mon[23721]: Deploying daemon grafana.smithi053 on smithi053 2024-09-13T05:20:37.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:37 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:38.527 INFO:teuthology.orchestra.run.smithi053.stderr:adjusted tunables profile to default 2024-09-13T05:20:38.896 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:38 smithi053 ceph-mon[23721]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:38.896 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:38 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/308242665' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T05:20:39.235 INFO:tasks.cephadm:Adding mon.smithi053 on smithi053 2024-09-13T05:20:39.236 INFO:tasks.cephadm:Adding mon.smithi110 on smithi110 2024-09-13T05:20:39.236 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi053:172.21.15.53=smithi053;smithi110:172.21.15.110=smithi110' 2024-09-13T05:20:39.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:39 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/308242665' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T05:20:39.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:39 smithi053 ceph-mon[23721]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T05:20:40.949 INFO:teuthology.orchestra.run.smithi110.stdout:Scheduled mon update... 2024-09-13T05:20:40.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:40 smithi053 ceph-mon[23721]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:41.548 DEBUG:teuthology.orchestra.run.smithi110:mon.smithi110> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi110.service 2024-09-13T05:20:41.551 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:20:41.551 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:20:42.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:41 smithi053 ceph-mon[23721]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi053:172.21.15.53=smithi053;smithi110:172.21.15.110=smithi110", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:20:42.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:41 smithi053 ceph-mon[23721]: Saving service mon spec with placement smithi053:172.21.15.53=smithi053;smithi110:172.21.15.110=smithi110;count:2 2024-09-13T05:20:42.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:41 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:42.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:41 smithi053 ceph-mon[23721]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:43.299 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:20:43.299 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:20:43.301 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:20:44.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:44 smithi053 ceph-mon[23721]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:44.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:44 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/872324911' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:20:44.869 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:20:44.870 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:20:46.642 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:20:46.643 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:20:46.645 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:20:46.745 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:46 smithi053 ceph-mon[23721]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:47.686 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:47 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/1252183968' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:20:48.215 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:20:48.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:20:48.983 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:48 smithi053 ceph-mon[23721]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:48.983 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:48 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:49.986 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:49 smithi053 ceph-mon[23721]: Deploying daemon node-exporter.smithi053 on smithi053 2024-09-13T05:20:50.049 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:20:50.049 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:20:50.052 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:20:50.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:50 smithi053 ceph-mon[23721]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:50.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:50 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/687490190' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:20:51.677 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:20:51.677 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:20:52.150 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:51 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:52.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:51 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T05:20:52.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:51 smithi053 ceph-mon[23721]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:20:53.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:52 smithi053 ceph-mon[23721]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T05:20:53.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:52 smithi053 ceph-mon[23721]: mgrmap e14: smithi053.ygzjut(active, since 44s) 2024-09-13T05:20:53.500 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:20:53.500 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:20:53.501 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:20:54.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:53 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/897811737' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:20:55.076 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:20:55.077 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:20:56.883 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:20:56.883 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:20:56.885 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:20:57.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:56 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/3504915859' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:20:58.456 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:20:58.456 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: Active manager daemon smithi053.ygzjut restarted 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: Activating manager daemon smithi053.ygzjut 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: mgrmap e15: smithi053.ygzjut(active, starting, since 0.0526809s) 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:20:59.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: [13/Sep/2024:05:20:58] ENGINE Bus STARTING 2024-09-13T05:20:59.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:20:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:00.138 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:21:00.139 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:21:00.142 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:21:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:00 smithi053 ceph-mon[23721]: [13/Sep/2024:05:20:59] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:21:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:00 smithi053 ceph-mon[23721]: [13/Sep/2024:05:20:59] ENGINE Bus STARTED 2024-09-13T05:21:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:00 smithi053 ceph-mon[23721]: mgrmap e16: smithi053.ygzjut(active, since 1.05511s) 2024-09-13T05:21:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:00 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:00.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:00 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/2771403721' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:01.751 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:21:01.751 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:21:02.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:01 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:02.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:01 smithi053 ceph-mon[23721]: mgrmap e17: smithi053.ygzjut(active, since 2s) 2024-09-13T05:21:03.858 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:21:03.858 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:21:03.860 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:21:04.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:03 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/4006028184' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:05.525 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:21:05.526 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:21:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:05 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:05 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:06.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:05 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd/host:smithi110", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:21:07.269 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:21:07.270 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:21:07.271 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:21:07.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd/host:smithi053", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:07.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:07 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/1923760805' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:08.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:08 smithi053 ceph-mon[23721]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:21:08.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:08 smithi053 ceph-mon[23721]: Deploying daemon crash.smithi110 on smithi110 2024-09-13T05:21:08.830 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:21:08.831 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:21:10.832 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:21:10.833 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:21:10.834 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:21:11.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:10 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/3275470942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:12.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:12.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:21:12.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T05:21:12.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:21:12.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:12.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:11 smithi053 ceph-mon[23721]: Deploying daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:21:12.481 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:21:12.482 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:21:14.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:14 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:14.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:14 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:14.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:14 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:21:14.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:14 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:14.855 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:21:14.855 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":1,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:19:06.284452Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T05:21:14.857 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 1 2024-09-13T05:21:15.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:15 smithi053 ceph-mon[23721]: Deploying daemon mon.smithi110 on smithi110 2024-09-13T05:21:15.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:15 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/676051584' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:17.665 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 podman[25782]: 2024-09-13 05:21:17.645072676 +0000 UTC m=+0.360048572 container start 2229074820b4b7d642e036545c5bc0b3616f26acb24530d713e89889f14134b1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, vendor=Red Hat, Inc., release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, name=centos-stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.buildah.version=1.19.8) 2024-09-13T05:21:17.665 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T05:21:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pidfile_write: ignore empty --pid-file 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: load: jerasure load: lrc 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: RocksDB version: 6.15.5 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Compile date Apr 18 2022 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: DB SUMMARY 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: DB Session ID: 27LBG2KO9RHFT0TRW0V5 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: CURRENT file: CURRENT 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi110/store.db dir, Total Num: 0, files: 2024-09-13T05:21:17.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi110/store.db: 000004.log size: 511 ; 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.error_if_exists: 0 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.create_if_missing: 0 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.paranoid_checks: 1 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.env: 0x562ed68c6860 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.fs: Posix File System 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.info_log: 0x562ed7f9bfa0 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.statistics: (nil) 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.use_fsync: 0 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_log_file_size: 0 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T05:21:17.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.allow_fallocate: 1 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.use_direct_reads: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.db_log_dir: 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi110/store.db 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T05:21:17.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T05:21:17.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T05:21:17.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.write_buffer_manager: 0x562ed808c330 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T05:21:18.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.unordered_write: 0 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.row_cache: None 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.wal_filter: None 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.preserve_deletes: 0 2024-09-13T05:21:18.001 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.two_write_queues: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.atomic_flush: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.log_readahead_size: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T05:21:18.002 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_background_jobs: 2 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_background_compactions: -1 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_subcompactions: 1 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_open_files: -1 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T05:21:18.003 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_background_flushes: -1 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Compression algorithms supported: 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kZSTD supported: 0 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kXpressCompression supported: 0 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kLZ4Compression supported: 1 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kBZip2Compression supported: 0 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kZlibCompression supported: 1 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: kSnappyCompression supported: 1 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi110/store.db/MANIFEST-000003 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T05:21:18.004 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.merge_operator: 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_filter: None 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562ed7f67d00) 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: cache_index_and_filter_blocks: 1 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: pin_top_level_index_and_filter: 1 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: index_type: 0 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: data_block_index_type: 0 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: index_shortening: 1 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T05:21:18.005 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: hash_index_allow_collision: 1 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: checksum: 1 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: no_block_cache: 0 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache: 0x562ed7fd2170 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache_name: BinnedLRUCache 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache_options: 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: capacity : 536870912 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: num_shard_bits : 4 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: strict_capacity_limit : 0 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: high_pri_pool_ratio: 0.000 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache_compressed: (nil) 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: persistent_cache: (nil) 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_size: 4096 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_size_deviation: 10 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_restart_interval: 16 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: index_block_restart_interval: 1 2024-09-13T05:21:18.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: metadata_block_size: 4096 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: partition_filters: 0 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: use_delta_encoding: 1 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: whole_key_filtering: 1 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: verify_compression: 0 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: read_amp_bytes_per_bit: 0 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: format_version: 4 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: enable_index_compression: 1 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_align: 0 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression: NoCompression 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T05:21:18.007 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.num_levels: 7 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T05:21:18.008 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T05:21:18.009 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T05:21:18.010 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.table_properties_collectors: 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.inplace_update_support: 0 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.bloom_locality: 0 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.max_successive_merges: 0 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T05:21:18.011 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.ttl: 2592000 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.enable_blob_files: false 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.min_blob_size: 0 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi110/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726204877671019, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T05:21:18.012 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726204877671319, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726204877, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "18392c62-3f6f-44aa-b1a2-80ac6a81e6fe", "db_session_id": "27LBG2KO9RHFT0TRW0V5"}} 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726204877671678, "job": 1, "event": "recovery_finished"} 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi110/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x562ed7fb8a80 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: DB pointer 0x562ed7fc8000 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** DB Stats ** 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:21:18.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:21:18.014 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T05:21:18.015 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.2 0.00 0.00 1 0.000 0 0 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T05:21:18.016 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110 does not exist in monmap, will attempt to join an existing cluster 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: using public_addr v2:172.21.15.110:0/0 -> [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: starting mon.smithi110 rank -1 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi110 fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(???) e0 preinit fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).mds e1 new map 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).mds e1 print_map 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: e1 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: legacy client fscid: -1 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: No filesystems configured 2024-09-13T05:21:18.017 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:18.018 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Deploying daemon crash.smithi053 on smithi053 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Deploying daemon grafana.smithi053 on smithi053 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.019 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/308242665' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/308242665' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi053:172.21.15.53=smithi053;smithi110:172.21.15.110=smithi110", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Saving service mon spec with placement smithi053:172.21.15.53=smithi053;smithi110:172.21.15.110=smithi110;count:2 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/872324911' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/1252183968' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Deploying daemon node-exporter.smithi053 on smithi053 2024-09-13T05:21:18.020 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/687490190' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14162 172.21.15.53:0/4178339427' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mgrmap e14: smithi053.ygzjut(active, since 44s) 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/897811737' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/3504915859' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Active manager daemon smithi053.ygzjut restarted 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Activating manager daemon smithi053.ygzjut 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mgrmap e15: smithi053.ygzjut(active, starting, since 0.0526809s) 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:21:18.021 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: [13/Sep/2024:05:20:58] ENGINE Bus STARTING 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: [13/Sep/2024:05:20:59] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:21:18.022 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: [13/Sep/2024:05:20:59] ENGINE Bus STARTED 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mgrmap e16: smithi053.ygzjut(active, since 1.05511s) 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/2771403721' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mgrmap e17: smithi053.ygzjut(active, since 2s) 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/4006028184' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd/host:smithi110", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd/host:smithi053", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:21:18.023 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/1923760805' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Deploying daemon crash.smithi110 on smithi110 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/3275470942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Deploying daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.024 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: Deploying daemon mon.smithi110 on smithi110 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/676051584' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 ceph-mon[25804]: mon.smithi110@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 bash[25782]: 2229074820b4b7d642e036545c5bc0b3616f26acb24530d713e89889f14134b1 2024-09-13T05:21:18.025 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:17 smithi110 systemd[1]: Started Ceph mon.smithi110 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:21:18.214 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T05:21:18.215 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T05:21:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: mon.smithi053 calling monitor election 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: mon.smithi110 calling monitor election 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:22.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: mon.smithi053 is new leader, mons smithi053,smithi110 in quorum (ranks 0,1) 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: monmap e2: 2 mons at {smithi053=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],smithi110=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: fsmap 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: mgrmap e17: smithi053.ygzjut(active, since 24s) 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: overall HEALTH_OK 2024-09-13T05:21:22.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:21:22.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:22.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:21:22.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:23.000 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:21:23.000 INFO:teuthology.orchestra.run.smithi110.stdout:{"epoch":2,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","modified":"2024-09-13T05:21:17.695187Z","created":"2024-09-13T05:19:06.284452Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi053","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:3300","nonce":0},{"type":"v1","addr":"172.21.15.53:6789","nonce":0}]},"addr":"172.21.15.53:6789/0","public_addr":"172.21.15.53:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi110","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:3300","nonce":0},{"type":"v1","addr":"172.21.15.110:6789","nonce":0}]},"addr":"172.21.15.110:6789/0","public_addr":"172.21.15.110:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T05:21:23.002 INFO:teuthology.orchestra.run.smithi110.stderr:dumped monmap epoch 2 2024-09-13T05:21:23.210 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:21:23.210 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: mon.smithi053 calling monitor election 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: mon.smithi110 calling monitor election 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: mon.smithi053 is new leader, mons smithi053,smithi110 in quorum (ranks 0,1) 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: monmap e2: 2 mons at {smithi053=[v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0],smithi110=[v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0]} 2024-09-13T05:21:23.211 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: fsmap 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: mgrmap e17: smithi053.ygzjut(active, since 24s) 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: overall HEALTH_OK 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/806612625' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:21:23.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:23.678 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T05:21:23.678 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T05:21:23.987 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:23 smithi053 ceph-mon[23721]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:23.987 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:23 smithi053 ceph-mon[23721]: Deploying daemon node-exporter.smithi110 on smithi110 2024-09-13T05:21:23.987 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:23 smithi053 ceph-mon[23721]: mgrmap e18: smithi053.ygzjut(active, since 24s), standbys: smithi110.rivldx 2024-09-13T05:21:23.987 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:23 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"}]: dispatch 2024-09-13T05:21:23.987 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:23 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/277017899' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:23.988 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:23 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:23 smithi110 ceph-mon[25804]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:23 smithi110 ceph-mon[25804]: Deploying daemon node-exporter.smithi110 on smithi110 2024-09-13T05:21:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:23 smithi110 ceph-mon[25804]: mgrmap e18: smithi053.ygzjut(active, since 24s), standbys: smithi110.rivldx 2024-09-13T05:21:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:23 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"}]: dispatch 2024-09-13T05:21:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:23 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/277017899' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T05:21:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:23 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:21:25.555 INFO:teuthology.orchestra.run.smithi053.stdout:# minimal ceph.conf for 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:21:25.555 INFO:teuthology.orchestra.run.smithi053.stdout:[global] 2024-09-13T05:21:25.555 INFO:teuthology.orchestra.run.smithi053.stdout: fsid = 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:21:25.555 INFO:teuthology.orchestra.run.smithi053.stdout: mon_host = [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] 2024-09-13T05:21:25.987 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:25 smithi110 ceph-mon[25804]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:25.988 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:25 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/699381110' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:25.996 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:25 smithi053 ceph-mon[23721]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:25.996 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:25 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/699381110' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:26.231 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T05:21:26.232 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:21:26.232 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T05:21:26.264 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:21:26.265 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:21:26.332 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-13T05:21:26.332 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T05:21:26.365 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-13T05:21:26.365 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:21:26.431 DEBUG:tasks.cephadm:set 0 configs 2024-09-13T05:21:26.431 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T05:21:26.432 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:21:26.432 DEBUG:teuthology.orchestra.run.smithi053:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T05:21:26.449 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T05:21:26.449 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_1 2024-09-13T05:21:26.506 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T05:21:26.506 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:26.506 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-13T05:21:26.506 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:26.507 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:26.507 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-09-13 05:20:48.325606062 +0000 2024-09-13T05:21:26.507 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-09-13 05:18:14.375537282 +0000 2024-09-13T05:21:26.507 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-09-13 05:18:14.375537282 +0000 2024-09-13T05:21:26.507 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-09-13 05:18:14.375537282 +0000 2024-09-13T05:21:26.507 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T05:21:26.570 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-09-13T05:21:26.571 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-09-13T05:21:26.571 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000109748 s, 4.7 MB/s 2024-09-13T05:21:26.572 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T05:21:26.630 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_2 2024-09-13T05:21:26.688 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T05:21:26.688 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:26.688 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-13T05:21:26.688 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:26.688 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:26.688 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-09-13 05:20:48.326606043 +0000 2024-09-13T05:21:26.688 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-09-13 05:18:14.696530931 +0000 2024-09-13T05:21:26.689 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-09-13 05:18:14.696530931 +0000 2024-09-13T05:21:26.689 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-09-13 05:18:14.696530931 +0000 2024-09-13T05:21:26.689 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T05:21:26.755 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-09-13T05:21:26.755 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-09-13T05:21:26.755 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000110747 s, 4.6 MB/s 2024-09-13T05:21:26.756 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T05:21:26.813 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_3 2024-09-13T05:21:26.869 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-09-13 05:20:48.326606043 +0000 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-09-13 05:18:15.023524461 +0000 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-09-13 05:18:15.023524461 +0000 2024-09-13T05:21:26.870 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-09-13 05:18:15.023524461 +0000 2024-09-13T05:21:26.870 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T05:21:26.933 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-09-13T05:21:26.933 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-09-13T05:21:26.933 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000110462 s, 4.6 MB/s 2024-09-13T05:21:26.935 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T05:21:26.992 DEBUG:teuthology.orchestra.run.smithi053:> stat /dev/vg_nvme/lv_4 2024-09-13T05:21:27.048 INFO:teuthology.orchestra.run.smithi053.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T05:21:27.048 INFO:teuthology.orchestra.run.smithi053.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:27.049 INFO:teuthology.orchestra.run.smithi053.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-13T05:21:27.049 INFO:teuthology.orchestra.run.smithi053.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:27.049 INFO:teuthology.orchestra.run.smithi053.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:27.049 INFO:teuthology.orchestra.run.smithi053.stdout:Access: 2024-09-13 05:20:48.327606023 +0000 2024-09-13T05:21:27.049 INFO:teuthology.orchestra.run.smithi053.stdout:Modify: 2024-09-13 05:18:15.341518170 +0000 2024-09-13T05:21:27.049 INFO:teuthology.orchestra.run.smithi053.stdout:Change: 2024-09-13 05:18:15.341518170 +0000 2024-09-13T05:21:27.049 INFO:teuthology.orchestra.run.smithi053.stdout: Birth: 2024-09-13 05:18:15.341518170 +0000 2024-09-13T05:21:27.049 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T05:21:27.112 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records in 2024-09-13T05:21:27.112 INFO:teuthology.orchestra.run.smithi053.stderr:1+0 records out 2024-09-13T05:21:27.112 INFO:teuthology.orchestra.run.smithi053.stderr:512 bytes copied, 0.000108218 s, 4.7 MB/s 2024-09-13T05:21:27.113 DEBUG:teuthology.orchestra.run.smithi053:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T05:21:27.170 DEBUG:teuthology.orchestra.run.smithi110:> set -ex 2024-09-13T05:21:27.170 DEBUG:teuthology.orchestra.run.smithi110:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T05:21:27.189 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T05:21:27.189 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_1 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-13 05:21:23.280321154 +0000 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-13 05:18:07.857027696 +0000 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-13 05:18:07.857027696 +0000 2024-09-13T05:21:27.248 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-13 05:18:07.857027696 +0000 2024-09-13T05:21:27.249 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T05:21:27.314 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-13T05:21:27.314 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-13T05:21:27.315 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000132765 s, 3.9 MB/s 2024-09-13T05:21:27.316 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T05:21:27.374 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_2 2024-09-13T05:21:27.432 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-13 05:21:23.281321137 +0000 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-13 05:18:08.184022349 +0000 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-13 05:18:08.184022349 +0000 2024-09-13T05:21:27.433 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-13 05:18:08.184022349 +0000 2024-09-13T05:21:27.433 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T05:21:27.497 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-13T05:21:27.498 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-13T05:21:27.498 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.000118967 s, 4.3 MB/s 2024-09-13T05:21:27.499 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T05:21:27.556 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_3 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-13 05:21:23.281321137 +0000 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-13 05:18:08.491017330 +0000 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-13 05:18:08.491017330 +0000 2024-09-13T05:21:27.612 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-13 05:18:08.491017330 +0000 2024-09-13T05:21:27.613 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T05:21:27.673 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:27 smithi110 ceph-mon[25804]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:27.673 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:27 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:27.673 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:27 smithi110 ceph-mon[25804]: Deploying daemon prometheus.smithi053 on smithi053 2024-09-13T05:21:27.675 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-13T05:21:27.676 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-13T05:21:27.676 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 9.1835e-05 s, 5.6 MB/s 2024-09-13T05:21:27.677 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T05:21:27.734 DEBUG:teuthology.orchestra.run.smithi110:> stat /dev/vg_nvme/lv_4 2024-09-13T05:21:27.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:27 smithi053 ceph-mon[23721]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:27.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:27 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:27.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:27 smithi053 ceph-mon[23721]: Deploying daemon prometheus.smithi053 on smithi053 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout:Access: 2024-09-13 05:21:23.281321137 +0000 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout:Modify: 2024-09-13 05:18:08.801012261 +0000 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout:Change: 2024-09-13 05:18:08.801012261 +0000 2024-09-13T05:21:27.791 INFO:teuthology.orchestra.run.smithi110.stdout: Birth: 2024-09-13 05:18:08.801012261 +0000 2024-09-13T05:21:27.792 DEBUG:teuthology.orchestra.run.smithi110:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T05:21:27.854 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records in 2024-09-13T05:21:27.854 INFO:teuthology.orchestra.run.smithi110.stderr:1+0 records out 2024-09-13T05:21:27.854 INFO:teuthology.orchestra.run.smithi110.stderr:512 bytes copied, 0.00011305 s, 4.5 MB/s 2024-09-13T05:21:27.856 DEBUG:teuthology.orchestra.run.smithi110:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T05:21:27.912 INFO:tasks.cephadm:Deploying osd.0 on smithi053 with /dev/vg_nvme/lv_4... 2024-09-13T05:21:27.912 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T05:21:28.794 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:28 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:28.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:28 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:29 smithi110 ceph-mon[25804]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:29.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:29 smithi053 ceph-mon[23721]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:31 smithi110 ceph-mon[25804]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:31.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:31 smithi053 ceph-mon[23721]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:33 smithi110 ceph-mon[25804]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:33 smithi053 ceph-mon[23721]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:35 smithi110 ceph-mon[25804]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:35.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:35 smithi053 ceph-mon[23721]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:37 smithi110 ceph-mon[25804]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:37.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:37 smithi053 ceph-mon[23721]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:39 smithi053 ceph-mon[23721]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:21:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:21:40.202 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:39 smithi110 ceph-mon[25804]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:40.203 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:40.203 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:21:40.203 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:40.203 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:21:40.941 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:21:40.963 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_4 2024-09-13T05:21:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:41 smithi110 ceph-mon[25804]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:41 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:21:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:41 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:21:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:41 smithi053 ceph-mon[23721]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:41 smithi053 ceph-mon[23721]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:21:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:41 smithi053 ceph-mon[23721]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:21:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:43 smithi110 ceph-mon[25804]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:43.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:43.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:21:43.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:21:43.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:43 smithi053 ceph-mon[23721]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:21:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:21:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:44.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:44 smithi110 ceph-mon[25804]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:21:44.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:44 smithi053 ceph-mon[23721]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:21:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:45 smithi110 ceph-mon[25804]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:45 smithi053 ceph-mon[23721]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:47 smithi110 ceph-mon[25804]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:47.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:47 smithi053 ceph-mon[23721]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3140876443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11bdd334-df82-49e9-86bc-e0da79b90761"}]: dispatch 2024-09-13T05:21:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3140876443' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11bdd334-df82-49e9-86bc-e0da79b90761"}]': finished 2024-09-13T05:21:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T05:21:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:21:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:48.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:48.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:48.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:21:48.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T05:21:48.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:48.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:48 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3130978525' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:21:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3140876443' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "11bdd334-df82-49e9-86bc-e0da79b90761"}]: dispatch 2024-09-13T05:21:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3140876443' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "11bdd334-df82-49e9-86bc-e0da79b90761"}]': finished 2024-09-13T05:21:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T05:21:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:21:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:48.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:21:48.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T05:21:48.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:48.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:48 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3130978525' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:21:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:49 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:21:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:49 smithi110 ceph-mon[25804]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:49 smithi110 ceph-mon[25804]: Reconfiguring mon.smithi053 (unknown last config time)... 2024-09-13T05:21:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:49 smithi110 ceph-mon[25804]: Reconfiguring daemon mon.smithi053 on smithi053 2024-09-13T05:21:49.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:49 smithi053 ceph-mon[23721]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:21:49.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:49 smithi053 ceph-mon[23721]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:49.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:49 smithi053 ceph-mon[23721]: Reconfiguring mon.smithi053 (unknown last config time)... 2024-09-13T05:21:49.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:49 smithi053 ceph-mon[23721]: Reconfiguring daemon mon.smithi053 on smithi053 2024-09-13T05:21:51.831 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:51 smithi053 ceph-mon[23721]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:51 smithi110 ceph-mon[25804]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:53.957 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:53 smithi053 ceph-mon[23721]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:53.957 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:53 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:53.957 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:53 smithi053 ceph-mon[23721]: Reconfiguring mgr.smithi053.ygzjut (unknown last config time)... 2024-09-13T05:21:53.957 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:53 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:21:53.958 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:53 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:21:53.958 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:53 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:53.958 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:53 smithi053 ceph-mon[23721]: Reconfiguring daemon mgr.smithi053.ygzjut on smithi053 2024-09-13T05:21:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:53 smithi110 ceph-mon[25804]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:53 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:53 smithi110 ceph-mon[25804]: Reconfiguring mgr.smithi053.ygzjut (unknown last config time)... 2024-09-13T05:21:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:53 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:21:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:53 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:21:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:53 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:53 smithi110 ceph-mon[25804]: Reconfiguring daemon mgr.smithi053.ygzjut on smithi053 2024-09-13T05:21:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:55 smithi110 ceph-mon[25804]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:55 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:55 smithi053 ceph-mon[23721]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:55 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:21:56.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:56 smithi053 ceph-mon[23721]: Reconfiguring alertmanager.smithi053 (dependencies changed)... 2024-09-13T05:21:56.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:56 smithi053 ceph-mon[23721]: Reconfiguring daemon alertmanager.smithi053 on smithi053 2024-09-13T05:21:56.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T05:21:56.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:56.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:56 smithi110 ceph-mon[25804]: Reconfiguring alertmanager.smithi053 (dependencies changed)... 2024-09-13T05:21:56.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:56 smithi110 ceph-mon[25804]: Reconfiguring daemon alertmanager.smithi053 on smithi053 2024-09-13T05:21:56.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T05:21:56.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:21:57.957 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:57 smithi053 ceph-mon[23721]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:57.957 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:57 smithi053 ceph-mon[23721]: Deploying daemon osd.0 on smithi053 2024-09-13T05:21:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:57 smithi110 ceph-mon[25804]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:57 smithi110 ceph-mon[25804]: Deploying daemon osd.0 on smithi053 2024-09-13T05:21:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:21:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:21:58.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:21:58.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:21:59.956 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:21:59 smithi053 ceph-mon[23721]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:21:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:21:59 smithi110 ceph-mon[25804]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:01 smithi110 ceph-mon[25804]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:01 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:01 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:22:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:01 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:01.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:01 smithi053 ceph-mon[23721]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:01.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:01 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:01.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:01 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:22:01.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:01 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:02.936 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:02 smithi053 ceph-mon[23721]: Reconfiguring crash.smithi053 (monmap changed)... 2024-09-13T05:22:02.936 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:02 smithi053 ceph-mon[23721]: Reconfiguring daemon crash.smithi053 on smithi053 2024-09-13T05:22:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:02 smithi110 ceph-mon[25804]: Reconfiguring crash.smithi053 (monmap changed)... 2024-09-13T05:22:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:02 smithi110 ceph-mon[25804]: Reconfiguring daemon crash.smithi053 on smithi053 2024-09-13T05:22:03.919 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:03 smithi053 ceph-mon[23721]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:03 smithi110 ceph-mon[25804]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:05.892 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:05 smithi053 ceph-mon[23721]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:05 smithi110 ceph-mon[25804]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:07.722 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:07 smithi053 ceph-mon[23721]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:07.723 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:07 smithi110 ceph-mon[25804]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:07 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:09 smithi110 ceph-mon[25804]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:09 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:09 smithi110 ceph-mon[25804]: Reconfiguring grafana.smithi053 (dependencies changed)... 2024-09-13T05:22:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:09 smithi110 ceph-mon[25804]: Reconfiguring daemon grafana.smithi053 on smithi053 2024-09-13T05:22:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:09 smithi110 ceph-mon[25804]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T05:22:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:09 smithi053 ceph-mon[23721]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:09 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:09 smithi053 ceph-mon[23721]: Reconfiguring grafana.smithi053 (dependencies changed)... 2024-09-13T05:22:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:09 smithi053 ceph-mon[23721]: Reconfiguring daemon grafana.smithi053 on smithi053 2024-09-13T05:22:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:09 smithi053 ceph-mon[23721]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T05:22:10.367 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 0 on host 'smithi053' 2024-09-13T05:22:11.007 DEBUG:teuthology.orchestra.run.smithi053:osd.0> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T05:22:11.009 INFO:tasks.cephadm:Deploying osd.1 on smithi053 with /dev/vg_nvme/lv_3... 2024-09-13T05:22:11.010 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T05:22:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:11 smithi110 ceph-mon[25804]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:11 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:11 smithi110 ceph-mon[25804]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T05:22:11.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:11 smithi110 ceph-mon[25804]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T05:22:11.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:11 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:11.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:11 smithi110 ceph-mon[25804]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:22:11.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-mon[23721]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:11.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:11.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-mon[23721]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T05:22:11.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-mon[23721]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T05:22:11.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:11.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-mon[23721]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:22:11.748 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[34214]: 2024-09-13T05:22:11.455+0000 7f9254667700 -1 osd.0 0 waiting for initial osdmap 2024-09-13T05:22:11.748 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:22:11 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[34214]: 2024-09-13T05:22:11.458+0000 7f924cffa700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:22:12.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:12 smithi110 ceph-mon[25804]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:22:12.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:12 smithi110 ceph-mon[25804]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T05:22:12.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:12 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:12.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:12 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:12.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:12 smithi053 ceph-mon[23721]: from='osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:22:12.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:12 smithi053 ceph-mon[23721]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T05:22:12.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:12 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:12.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:12 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:13.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:13 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:22:13.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:13 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:22:13.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:13 smithi110 ceph-mon[25804]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:13.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:13 smithi110 ceph-mon[25804]: osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571] boot 2024-09-13T05:22:13.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:13 smithi110 ceph-mon[25804]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T05:22:13.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:13 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:13 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:22:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:13 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:22:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:13 smithi053 ceph-mon[23721]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T05:22:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:13 smithi053 ceph-mon[23721]: osd.0 [v2:172.21.15.53:6802/1675242571,v1:172.21.15.53:6803/1675242571] boot 2024-09-13T05:22:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:13 smithi053 ceph-mon[23721]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T05:22:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:13 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:22:14.715 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:14 smithi053 ceph-mon[23721]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T05:22:14.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:14 smithi110 ceph-mon[25804]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T05:22:15.730 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:15 smithi053 ceph-mon[23721]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:15 smithi110 ceph-mon[25804]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:16.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:16 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:16.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:16 smithi053 ceph-mon[23721]: Reconfiguring crash.smithi110 (monmap changed)... 2024-09-13T05:22:16.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:16 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:22:16.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:16 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:16.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:16 smithi053 ceph-mon[23721]: Reconfiguring daemon crash.smithi110 on smithi110 2024-09-13T05:22:16.863 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:16 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:16.864 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:16 smithi110 ceph-mon[25804]: Reconfiguring crash.smithi110 (monmap changed)... 2024-09-13T05:22:16.864 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:16 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T05:22:16.864 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:16 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:16.864 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:16 smithi110 ceph-mon[25804]: Reconfiguring daemon crash.smithi110 on smithi110 2024-09-13T05:22:16.992 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:22:17.011 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_3 2024-09-13T05:22:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:17 smithi053 ceph-mon[23721]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:17 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:17 smithi053 ceph-mon[23721]: Reconfiguring mgr.smithi110.rivldx (monmap changed)... 2024-09-13T05:22:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:17 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:22:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:17 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:22:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:17 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:17 smithi053 ceph-mon[23721]: Reconfiguring daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:22:18.215 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:17 smithi110 ceph-mon[25804]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:18.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:18.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:17 smithi110 ceph-mon[25804]: Reconfiguring mgr.smithi110.rivldx (monmap changed)... 2024-09-13T05:22:18.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:22:18.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:22:18.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:17 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:18.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:17 smithi110 ceph-mon[25804]: Reconfiguring daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:22:19.438 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:19.438 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: Reconfiguring mon.smithi110 (monmap changed)... 2024-09-13T05:22:19.438 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:22:19.438 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T05:22:19.438 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:19.438 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: Reconfiguring daemon mon.smithi110 on smithi110 2024-09-13T05:22:19.439 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:19.439 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:22:19.439 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:22:19.439 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: Reconfiguring mon.smithi110 (monmap changed)... 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: Reconfiguring daemon mon.smithi110 on smithi110 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:19.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:22:19.489 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:22:19.489 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:20.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:20 smithi053 ceph-mon[23721]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:22:20.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:20 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:20.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:20 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:22:20.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:20 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:22:20.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:20 smithi110 ceph-mon[25804]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:22:20.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:20 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:20.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:20 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:22:20.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:20 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:22:21.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:21 smithi053 ceph-mon[23721]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:21.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:21 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1319077618' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c4006a9-cb6a-4f0d-a328-7d56c363afe4"}]: dispatch 2024-09-13T05:22:21.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:21 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1319077618' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c4006a9-cb6a-4f0d-a328-7d56c363afe4"}]': finished 2024-09-13T05:22:21.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:21 smithi053 ceph-mon[23721]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T05:22:21.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:21 smithi110 ceph-mon[25804]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:21 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1319077618' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c4006a9-cb6a-4f0d-a328-7d56c363afe4"}]: dispatch 2024-09-13T05:22:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:21 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1319077618' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c4006a9-cb6a-4f0d-a328-7d56c363afe4"}]': finished 2024-09-13T05:22:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:21 smithi110 ceph-mon[25804]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T05:22:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:22 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2160130397' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:22:22.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:22 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2160130397' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:22:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:23 smithi053 ceph-mon[23721]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:23 smithi110 ceph-mon[25804]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:25.657 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:25 smithi053 ceph-mon[23721]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:25 smithi110 ceph-mon[25804]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:27.636 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:27 smithi053 ceph-mon[23721]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:27 smithi110 ceph-mon[25804]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:28.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:28 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T05:22:28.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:28 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:28.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:28 smithi110 ceph-mon[25804]: Deploying daemon osd.1 on smithi053 2024-09-13T05:22:28.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:28 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T05:22:28.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:28 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:28.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:28 smithi053 ceph-mon[23721]: Deploying daemon osd.1 on smithi053 2024-09-13T05:22:29.641 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:29 smithi053 ceph-mon[23721]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:29 smithi110 ceph-mon[25804]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:31 smithi110 ceph-mon[25804]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:31.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:31 smithi053 ceph-mon[23721]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:34.156 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:33 smithi053 ceph-mon[23721]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:34.156 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:33 smithi110 ceph-mon[25804]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:35.670 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 1 on host 'smithi053' 2024-09-13T05:22:35.971 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.53:9093"}]: dispatch 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:35.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:22:36.241 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.53:9093"}]: dispatch 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:36.242 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:22:36.315 DEBUG:teuthology.orchestra.run.smithi053:osd.1> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T05:22:36.318 INFO:tasks.cephadm:Deploying osd.2 on smithi053 with /dev/vg_nvme/lv_2... 2024-09-13T05:22:36.318 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T05:22:36.953 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:36 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.53:9093"}]: dispatch 2024-09-13T05:22:36.953 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:36 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:22:36.953 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:36 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-09-13T05:22:36.953 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:36 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:22:36.953 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:36 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-09-13T05:22:36.953 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:36 smithi053 ceph-mon[23721]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T05:22:37.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:36 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.53:9093"}]: dispatch 2024-09-13T05:22:37.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:36 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:22:37.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:36 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.53:3000"}]: dispatch 2024-09-13T05:22:37.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:36 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:22:37.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:36 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.53:9095"}]: dispatch 2024-09-13T05:22:37.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:36 smithi110 ceph-mon[25804]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T05:22:38.168 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-mon[23721]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:38.169 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-mon[23721]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T05:22:38.169 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-mon[23721]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T05:22:38.169 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:38.169 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-mon[23721]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:22:38.169 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:38.169 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[37057]: 2024-09-13T05:22:37.868+0000 7fb9f16b1700 -1 osd.1 0 waiting for initial osdmap 2024-09-13T05:22:38.169 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:22:37 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[37057]: 2024-09-13T05:22:37.871+0000 7fb9eb847700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:22:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:37 smithi110 ceph-mon[25804]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:37 smithi110 ceph-mon[25804]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T05:22:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:37 smithi110 ceph-mon[25804]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T05:22:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:37 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:37 smithi110 ceph-mon[25804]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:22:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:37 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:39.184 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:38 smithi053 ceph-mon[23721]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:22:39.184 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:38 smithi053 ceph-mon[23721]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T05:22:39.184 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:38 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:39.184 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:38 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:38 smithi110 ceph-mon[25804]: from='osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:22:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:38 smithi110 ceph-mon[25804]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T05:22:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:38 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:38 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:40.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:39 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:22:40.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:39 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:22:40.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:39 smithi110 ceph-mon[25804]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:40.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:40.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:39 smithi110 ceph-mon[25804]: osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156] boot 2024-09-13T05:22:40.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:39 smithi110 ceph-mon[25804]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T05:22:40.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:39 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:22:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:39 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:22:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:39 smithi053 ceph-mon[23721]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T05:22:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:39 smithi053 ceph-mon[23721]: osd.1 [v2:172.21.15.53:6810/3213587156,v1:172.21.15.53:6811/3213587156] boot 2024-09-13T05:22:40.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:39 smithi053 ceph-mon[23721]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T05:22:40.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:22:40.831 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:22:40.856 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_2 2024-09-13T05:22:41.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:40 smithi053 ceph-mon[23721]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T05:22:41.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:40 smithi110 ceph-mon[25804]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T05:22:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:41 smithi110 ceph-mon[25804]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:42.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:41 smithi053 ceph-mon[23721]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:44.155 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:43 smithi053 ceph-mon[23721]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:44.156 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:22:44.156 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:22:44.156 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:44.156 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:43 smithi110 ceph-mon[25804]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:22:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:22:44.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:44.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:45.126 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:44 smithi053 ceph-mon[23721]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:22:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:44 smithi110 ceph-mon[25804]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:22:45.999 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:45 smithi053 ceph-mon[23721]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:45 smithi110 ceph-mon[25804]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:47.145 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:46 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1611573341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beefd695-6c5d-419d-bf38-acde87bfe835"}]: dispatch 2024-09-13T05:22:47.146 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:46 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1611573341' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "beefd695-6c5d-419d-bf38-acde87bfe835"}]': finished 2024-09-13T05:22:47.146 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:46 smithi110 ceph-mon[25804]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T05:22:47.146 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:46 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:22:47.146 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:46 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2562041973' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:22:47.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:46 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1611573341' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "beefd695-6c5d-419d-bf38-acde87bfe835"}]: dispatch 2024-09-13T05:22:47.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:46 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1611573341' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "beefd695-6c5d-419d-bf38-acde87bfe835"}]': finished 2024-09-13T05:22:47.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:46 smithi053 ceph-mon[23721]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T05:22:47.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:46 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:22:47.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:46 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2562041973' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:22:48.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:47 smithi110 ceph-mon[25804]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:48.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:47 smithi053 ceph-mon[23721]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:50.172 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:49 smithi053 ceph-mon[23721]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:50.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:49 smithi110 ceph-mon[25804]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:52.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:51 smithi053 ceph-mon[23721]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:52.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:52.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:22:52.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:22:52.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:52.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:52.194 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:22:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:51 smithi110 ceph-mon[25804]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:22:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:22:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:22:52.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:52 smithi053 ceph-mon[23721]: Detected new or changed devices on smithi053 2024-09-13T05:22:52.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:52 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi053 to 6434M 2024-09-13T05:22:52.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:52 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:52.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:52 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:22:53.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:52 smithi110 ceph-mon[25804]: Detected new or changed devices on smithi053 2024-09-13T05:22:53.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:52 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi053 to 6434M 2024-09-13T05:22:53.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:52 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:53.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:52 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:22:54.055 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:53 smithi053 ceph-mon[23721]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:54.055 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:53 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:54.056 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:53 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T05:22:54.056 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:53 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:54.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:53 smithi110 ceph-mon[25804]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:54.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:53 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:22:54.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:53 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T05:22:54.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:53 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:22:54.984 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:54 smithi053 ceph-mon[23721]: Deploying daemon osd.2 on smithi053 2024-09-13T05:22:55.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:54 smithi110 ceph-mon[25804]: Deploying daemon osd.2 on smithi053 2024-09-13T05:22:56.190 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:55 smithi053 ceph-mon[23721]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:56.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:55 smithi110 ceph-mon[25804]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:58.185 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:57 smithi053 ceph-mon[23721]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:58.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:57 smithi110 ceph-mon[25804]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:58.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:22:58.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:22:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:22:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:22:59.988 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:59 smithi053 ceph-mon[23721]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:22:59.988 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:22:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:59 smithi110 ceph-mon[25804]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:22:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:02.045 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:01 smithi053 ceph-mon[23721]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:01 smithi110 ceph-mon[25804]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:03 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:03 smithi053 ceph-mon[23721]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:03 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:03 smithi110 ceph-mon[25804]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:04.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:04.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:04.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:04.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:04.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:05.688 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 2 on host 'smithi053' 2024-09-13T05:23:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:05 smithi110 ceph-mon[25804]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:05 smithi110 ceph-mon[25804]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T05:23:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:05 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:05 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:05 smithi053 ceph-mon[23721]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:05 smithi053 ceph-mon[23721]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T05:23:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:05 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:05 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:06.345 DEBUG:teuthology.orchestra.run.smithi053:osd.2> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T05:23:06.347 INFO:tasks.cephadm:Deploying osd.3 on smithi053 with /dev/vg_nvme/lv_1... 2024-09-13T05:23:06.348 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T05:23:06.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:06 smithi053 ceph-mon[23721]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T05:23:06.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:06 smithi053 ceph-mon[23721]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T05:23:06.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:06 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:06.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:06 smithi053 ceph-mon[23721]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:23:06.894 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:06 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[40874]: 2024-09-13T05:23:06.727+0000 7fde57d3f700 -1 osd.2 0 waiting for initial osdmap 2024-09-13T05:23:06.894 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:06 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[40874]: 2024-09-13T05:23:06.731+0000 7fde50ed3700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:23:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:06 smithi110 ceph-mon[25804]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T05:23:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:06 smithi110 ceph-mon[25804]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T05:23:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:06 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:06 smithi110 ceph-mon[25804]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:23:07.983 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:07 smithi053 ceph-mon[23721]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:07.983 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:07 smithi053 ceph-mon[23721]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:23:07.983 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:07 smithi053 ceph-mon[23721]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T05:23:07.984 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:07.984 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:07 smithi110 ceph-mon[25804]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:07 smithi110 ceph-mon[25804]: from='osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:23:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:07 smithi110 ceph-mon[25804]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T05:23:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:07 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:07 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:09.087 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:08 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:23:09.087 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:08 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:23:09.087 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:08 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:09.087 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:08 smithi053 ceph-mon[23721]: osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317] boot 2024-09-13T05:23:09.088 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:08 smithi053 ceph-mon[23721]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T05:23:09.088 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:08 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:09.088 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:08 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T05:23:09.093 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:23:09.112 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi053:vg_nvme/lv_1 2024-09-13T05:23:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:08 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:23:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:08 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:23:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:08 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:08 smithi110 ceph-mon[25804]: osd.2 [v2:172.21.15.53:6818/366341317,v1:172.21.15.53:6819/366341317] boot 2024-09-13T05:23:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:08 smithi110 ceph-mon[25804]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T05:23:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:08 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:23:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:08 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T05:23:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:09 smithi110 ceph-mon[25804]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:09 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T05:23:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:09 smithi110 ceph-mon[25804]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T05:23:10.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:09 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T05:23:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:09 smithi053 ceph-mon[23721]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-13T05:23:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:09 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T05:23:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:09 smithi053 ceph-mon[23721]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T05:23:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:09 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T05:23:11.095 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T05:23:11.095 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:10 smithi053 ceph-mon[23721]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T05:23:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T05:23:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:10 smithi110 ceph-mon[25804]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T05:23:11.453 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:23:11 smithi053 sudo[42198]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T05:23:11.453 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:23:11 smithi053 sudo[42198]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:11.453 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:23:11 smithi053 sudo[42198]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:12.097 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:11 smithi053 ceph-mon[23721]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:12.097 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:11 smithi053 ceph-mon[23721]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T05:23:12.097 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:23:12.097 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:23:12.097 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:12.097 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:11 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:12.097 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:23:11 smithi053 sudo[42250]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T05:23:12.097 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:23:11 smithi053 sudo[42250]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:12.097 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:23:11 smithi053 sudo[42250]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:11 smithi110 ceph-mon[25804]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:11 smithi110 ceph-mon[25804]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T05:23:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:11 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:23:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:11 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:23:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:11 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:11 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:12.439 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:23:12 smithi053 sudo[42345]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T05:23:12.440 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:23:12 smithi053 sudo[42345]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:12.440 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:23:12 smithi053 sudo[42345]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:12.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:12 smithi053 ceph-mon[23721]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:23:12.998 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:23:12 smithi053 sudo[42394]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T05:23:12.998 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:23:12 smithi053 sudo[42394]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:12.998 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:23:12 smithi053 sudo[42394]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:12 smithi110 ceph-mon[25804]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi053:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:23:13.390 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:13 smithi053 sudo[42414]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T05:23:13.390 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:13 smithi053 sudo[42414]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:13.391 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:13 smithi053 sudo[42414]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:13.681 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:13 smithi053 sudo[42420]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T05:23:13.681 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:13 smithi053 sudo[42420]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:13.681 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:23:13 smithi053 sudo[42420]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:14.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 sudo[42471]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T05:23:14.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 sudo[42471]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:14.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 sudo[42471]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:14.738 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 sudo[42478]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T05:23:14.738 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 sudo[42478]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:14.738 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 sudo[42478]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:14.738 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 ceph-mon[23721]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:14.738 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:14 smithi053 ceph-mon[23721]: mgrmap e19: smithi053.ygzjut(active, since 2m), standbys: smithi110.rivldx 2024-09-13T05:23:14.771 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:14 smithi110 ceph-mon[25804]: pgmap v77: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:14.772 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:14 smithi110 ceph-mon[25804]: mgrmap e19: smithi053.ygzjut(active, since 2m), standbys: smithi110.rivldx 2024-09-13T05:23:14.772 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:14 smithi110 sudo[28116]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T05:23:14.772 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:14 smithi110 sudo[28116]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:14.772 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:14 smithi110 sudo[28116]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:15.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 sudo[28119]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T05:23:15.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 sudo[28119]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T05:23:15.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 sudo[28119]: pam_unix(sudo:session): session closed for user root 2024-09-13T05:23:15.710 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T05:23:15.710 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1623737084' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7dd6a140-c882-467e-9c50-fe9f9a1f57ae"}]: dispatch 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1623737084' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7dd6a140-c882-467e-9c50-fe9f9a1f57ae"}]': finished 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T05:23:15.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T05:23:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1623737084' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7dd6a140-c882-467e-9c50-fe9f9a1f57ae"}]: dispatch 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1623737084' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7dd6a140-c882-467e-9c50-fe9f9a1f57ae"}]': finished 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: osdmap e23: 4 total, 3 up, 4 in 2024-09-13T05:23:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:16.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:16 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/4071770999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:23:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:16 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/4071770999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:23:17.895 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:17 smithi110 ceph-mon[25804]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:17 smithi053 ceph-mon[23721]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:19.683 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:19 smithi053 ceph-mon[23721]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:19 smithi110 ceph-mon[25804]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:21.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 159 KiB/s wr, 11 op/s 2024-09-13T05:23:21.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:21.726 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.7 KiB/s rd, 159 KiB/s wr, 11 op/s 2024-09-13T05:23:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:22 smithi110 ceph-mon[25804]: Detected new or changed devices on smithi053 2024-09-13T05:23:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:22 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi053 to 4289M 2024-09-13T05:23:22.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:22 smithi053 ceph-mon[23721]: Detected new or changed devices on smithi053 2024-09-13T05:23:22.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:22 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi053 to 4289M 2024-09-13T05:23:23.855 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:23 smithi053 ceph-mon[23721]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 113 KiB/s wr, 8 op/s 2024-09-13T05:23:23.856 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:23 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T05:23:23.856 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:23 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:23 smithi110 ceph-mon[25804]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 113 KiB/s wr, 8 op/s 2024-09-13T05:23:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:23 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T05:23:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:23 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:24.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:24 smithi053 ceph-mon[23721]: Deploying daemon osd.3 on smithi053 2024-09-13T05:23:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:24 smithi110 ceph-mon[25804]: Deploying daemon osd.3 on smithi053 2024-09-13T05:23:25.783 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:25 smithi053 ceph-mon[23721]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.0 KiB/s rd, 88 KiB/s wr, 6 op/s 2024-09-13T05:23:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:25 smithi110 ceph-mon[25804]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.0 KiB/s rd, 88 KiB/s wr, 6 op/s 2024-09-13T05:23:27.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:27 smithi053 ceph-mon[23721]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 72 KiB/s wr, 5 op/s 2024-09-13T05:23:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:27 smithi110 ceph-mon[25804]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 72 KiB/s wr, 5 op/s 2024-09-13T05:23:29.872 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:29 smithi053 ceph-mon[23721]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T05:23:29.872 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:29.872 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:29 smithi110 ceph-mon[25804]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T05:23:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:30.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:30 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:30.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:30 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:30.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:30 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:30.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:30 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:30 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:30 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:30 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:30 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:31 smithi110 ceph-mon[25804]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T05:23:31.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:31 smithi053 ceph-mon[23721]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T05:23:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:33 smithi110 ceph-mon[25804]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:33 smithi110 ceph-mon[25804]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T05:23:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:33 smithi053 ceph-mon[23721]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:33.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:33 smithi053 ceph-mon[23721]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T05:23:34.063 INFO:teuthology.orchestra.run.smithi053.stdout:Created osd(s) 3 on host 'smithi053' 2024-09-13T05:23:34.669 DEBUG:teuthology.orchestra.run.smithi053:osd.3> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T05:23:34.671 INFO:tasks.cephadm:Deploying osd.4 on smithi110 with /dev/vg_nvme/lv_4... 2024-09-13T05:23:34.672 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T05:23:34.982 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:34 smithi110 ceph-mon[25804]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T05:23:34.982 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:34 smithi110 ceph-mon[25804]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T05:23:34.983 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:34 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:34.983 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:34 smithi110 ceph-mon[25804]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:23:34.983 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:34 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:34.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:34 smithi053 ceph-mon[23721]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T05:23:34.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:34 smithi053 ceph-mon[23721]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T05:23:34.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:34 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:34.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:34 smithi053 ceph-mon[23721]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]: dispatch 2024-09-13T05:23:34.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:34 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:34.998 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:23:34 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[44687]: 2024-09-13T05:23:34.702+0000 7fd38e859700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:23:35.948 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:35 smithi110 ceph-mon[25804]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:35.948 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:35 smithi110 ceph-mon[25804]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:23:35.948 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:35 smithi110 ceph-mon[25804]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T05:23:35.948 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:35.948 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:35 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:35.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:35 smithi053 ceph-mon[23721]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:35.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:35 smithi053 ceph-mon[23721]: from='osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]}]': finished 2024-09-13T05:23:35.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:35 smithi053 ceph-mon[23721]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T05:23:35.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:35.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:35 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:36.955 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:36 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:23:36.956 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:36 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:23:36.956 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:36 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:36.956 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:36 smithi053 ceph-mon[23721]: osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796] boot 2024-09-13T05:23:36.956 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:36 smithi053 ceph-mon[23721]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T05:23:36.956 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:36 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:36.965 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:36 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:23:36.965 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:36 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:23:36.965 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:36 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:36.966 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:36 smithi110 ceph-mon[25804]: osd.3 [v2:172.21.15.53:6826/3452758796,v1:172.21.15.53:6827/3452758796] boot 2024-09-13T05:23:36.966 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:36 smithi110 ceph-mon[25804]: osdmap e26: 4 total, 4 up, 4 in 2024-09-13T05:23:36.966 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:36 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:23:36.970 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:23:36.989 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_4 2024-09-13T05:23:37.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:37 smithi053 ceph-mon[23721]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:37.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:37 smithi053 ceph-mon[23721]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T05:23:37.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:37 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:37.929 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:37 smithi110 ceph-mon[25804]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T05:23:37.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:37 smithi110 ceph-mon[25804]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T05:23:37.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:37 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:39.888 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:39 smithi110 ceph-mon[25804]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:39.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:39 smithi110 ceph-mon[25804]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:23:39.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:23:39.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:23:39.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:39.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:39 smithi053 ceph-mon[23721]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:39.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:39 smithi053 ceph-mon[23721]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:23:39.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:23:39.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:23:39.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi053 to 3217M 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:41.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:41.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:41.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:41.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/4161770917' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "988194ce-8aad-4b6c-8269-1c1a6fdabb0f"}]: dispatch 2024-09-13T05:23:41.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "988194ce-8aad-4b6c-8269-1c1a6fdabb0f"}]: dispatch 2024-09-13T05:23:41.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "988194ce-8aad-4b6c-8269-1c1a6fdabb0f"}]': finished 2024-09-13T05:23:41.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T05:23:41.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:41 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi053 to 3217M 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/4161770917' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "988194ce-8aad-4b6c-8269-1c1a6fdabb0f"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "988194ce-8aad-4b6c-8269-1c1a6fdabb0f"}]: dispatch 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "988194ce-8aad-4b6c-8269-1c1a6fdabb0f"}]': finished 2024-09-13T05:23:41.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: osdmap e28: 5 total, 4 up, 5 in 2024-09-13T05:23:41.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:41 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:42.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:42 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/270776004' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:23:42.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:42 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/270776004' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:23:43.881 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:43 smithi110 ceph-mon[25804]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:43 smithi053 ceph-mon[23721]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:46.131 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:45 smithi110 ceph-mon[25804]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:46.131 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:45 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:46.131 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:45 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:46.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:45 smithi053 ceph-mon[23721]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:46.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:45 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:46.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:45 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:48.237 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:47 smithi110 ceph-mon[25804]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:48.238 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:47 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T05:23:48.238 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:47 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:48.238 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:47 smithi110 ceph-mon[25804]: Deploying daemon osd.4 on smithi110 2024-09-13T05:23:48.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:47 smithi053 ceph-mon[23721]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:48.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:47 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T05:23:48.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:47 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:48.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:47 smithi053 ceph-mon[23721]: Deploying daemon osd.4 on smithi110 2024-09-13T05:23:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:49 smithi110 ceph-mon[25804]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:50.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:49 smithi053 ceph-mon[23721]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:52.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:51 smithi110 ceph-mon[25804]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:52.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:52.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:52.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:52.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:52.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:51 smithi053 ceph-mon[23721]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:52.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:52.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:52.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:52.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:54.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:53 smithi110 ceph-mon[25804]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:54.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:53 smithi053 ceph-mon[23721]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:55.274 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 4 on host 'smithi110' 2024-09-13T05:23:55.811 DEBUG:teuthology.orchestra.run.smithi110:osd.4> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T05:23:55.814 INFO:tasks.cephadm:Deploying osd.5 on smithi110 with /dev/vg_nvme/lv_3... 2024-09-13T05:23:55.814 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T05:23:56.126 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:56.126 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='osd.4 [v2:172.21.15.110:6800/3538242982,v1:172.21.15.110:6801/3538242982]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T05:23:56.127 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T05:23:56.127 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:56.127 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:56.127 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:56.127 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:56.127 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:56.127 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:55 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:56.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='osd.4 [v2:172.21.15.110:6800/3538242982,v1:172.21.15.110:6801/3538242982]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:23:56.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:55 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:23:57.148 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:56 smithi110 ceph-mon[25804]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T05:23:57.148 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:56 smithi110 ceph-mon[25804]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T05:23:57.148 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:57.148 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:56 smithi110 ceph-mon[25804]: from='osd.4 [v2:172.21.15.110:6800/3538242982,v1:172.21.15.110:6801/3538242982]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:23:57.148 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:56 smithi110 ceph-mon[25804]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:23:57.149 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:23:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[30044]: 2024-09-13T05:23:56.878+0000 7fe88f25a700 -1 osd.4 0 waiting for initial osdmap 2024-09-13T05:23:57.149 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:23:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[30044]: 2024-09-13T05:23:56.882+0000 7fe88a3f2700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:23:57.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:56 smithi053 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T05:23:57.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:56 smithi053 ceph-mon[23721]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T05:23:57.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:57.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:56 smithi053 ceph-mon[23721]: from='osd.4 [v2:172.21.15.110:6800/3538242982,v1:172.21.15.110:6801/3538242982]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:23:57.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:56 smithi053 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:23:58.132 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:57 smithi110 ceph-mon[25804]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:58.132 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:57 smithi110 ceph-mon[25804]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:23:58.132 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:57 smithi110 ceph-mon[25804]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T05:23:58.132 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:57 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:58.132 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:57 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:57 smithi053 ceph-mon[23721]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:23:58.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:57 smithi053 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:23:58.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:57 smithi053 ceph-mon[23721]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T05:23:58.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:57 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:58.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:57 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:23:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:23:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: osd.4 [v2:172.21.15.110:6800/3538242982,v1:172.21.15.110:6801/3538242982] boot 2024-09-13T05:23:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T05:23:59.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:59.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:23:59.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:23:59.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:23:59.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:23:59.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:59.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: osd.4 [v2:172.21.15.110:6800/3538242982,v1:172.21.15.110:6801/3538242982] boot 2024-09-13T05:23:59.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: osdmap e31: 5 total, 5 up, 5 in 2024-09-13T05:23:59.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:23:59.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:23:59.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:24:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:59 smithi110 ceph-mon[25804]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:24:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:59 smithi110 ceph-mon[25804]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T05:24:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:00.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:23:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:00.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:59 smithi053 ceph-mon[23721]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T05:24:00.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:59 smithi053 ceph-mon[23721]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T05:24:00.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:00.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:00.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:00.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:23:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:00.871 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:24:00.891 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_3 2024-09-13T05:24:01.188 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:00 smithi110 ceph-mon[25804]: Detected new or changed devices on smithi110 2024-09-13T05:24:01.188 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:00 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi110 to 15936M 2024-09-13T05:24:01.188 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:00 smithi110 ceph-mon[25804]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T05:24:01.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:00 smithi053 ceph-mon[23721]: Detected new or changed devices on smithi110 2024-09-13T05:24:01.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:00 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi110 to 15936M 2024-09-13T05:24:01.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:00 smithi053 ceph-mon[23721]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T05:24:02.071 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:01 smithi110 ceph-mon[25804]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:02.071 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:01 smithi110 ceph-mon[25804]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T05:24:02.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:01 smithi053 ceph-mon[23721]: pgmap v112: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:02.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:01 smithi053 ceph-mon[23721]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T05:24:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:02 smithi110 ceph-mon[25804]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T05:24:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:24:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:24:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:02 smithi053 ceph-mon[23721]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T05:24:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:24:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:24:03.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:04.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:03 smithi110 ceph-mon[25804]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:04.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:03 smithi110 ceph-mon[25804]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:24:04.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:03 smithi053 ceph-mon[23721]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:04.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:03 smithi053 ceph-mon[23721]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:24:06.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:05 smithi110 ceph-mon[25804]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:06.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:05 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/4159766132' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7e15755a-7760-4cbd-ac69-400b6de3f247"}]: dispatch 2024-09-13T05:24:06.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:05 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7e15755a-7760-4cbd-ac69-400b6de3f247"}]: dispatch 2024-09-13T05:24:06.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:05 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7e15755a-7760-4cbd-ac69-400b6de3f247"}]': finished 2024-09-13T05:24:06.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:05 smithi110 ceph-mon[25804]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T05:24:06.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:05 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:06.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:05 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/1107767356' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:24:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:05 smithi053 ceph-mon[23721]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:05 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/4159766132' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7e15755a-7760-4cbd-ac69-400b6de3f247"}]: dispatch 2024-09-13T05:24:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:05 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7e15755a-7760-4cbd-ac69-400b6de3f247"}]: dispatch 2024-09-13T05:24:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:05 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7e15755a-7760-4cbd-ac69-400b6de3f247"}]': finished 2024-09-13T05:24:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:05 smithi053 ceph-mon[23721]: osdmap e36: 6 total, 5 up, 6 in 2024-09-13T05:24:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:05 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:05 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/1107767356' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:24:08.013 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:07 smithi110 ceph-mon[25804]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T05:24:08.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:07 smithi053 ceph-mon[23721]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T05:24:10.205 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:09 smithi110 ceph-mon[25804]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T05:24:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:09 smithi053 ceph-mon[23721]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-09-13T05:24:11.201 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T05:24:11.201 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:11.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T05:24:11.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:12.207 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:11 smithi110 ceph-mon[25804]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T05:24:12.207 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:11 smithi110 ceph-mon[25804]: Deploying daemon osd.5 on smithi110 2024-09-13T05:24:12.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:11 smithi053 ceph-mon[23721]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-13T05:24:12.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:11 smithi053 ceph-mon[23721]: Deploying daemon osd.5 on smithi110 2024-09-13T05:24:14.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:13 smithi110 ceph-mon[25804]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:24:14.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:13 smithi053 ceph-mon[23721]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:24:16.138 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:15 smithi110 ceph-mon[25804]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:24:16.139 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:16.139 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:16.139 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:16.139 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:15 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:16.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:15 smithi053 ceph-mon[23721]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:24:16.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:16.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:16.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:16.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:15 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:17 smithi110 ceph-mon[25804]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:18.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:17 smithi053 ceph-mon[23721]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:18.607 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 5 on host 'smithi110' 2024-09-13T05:24:19.136 DEBUG:teuthology.orchestra.run.smithi110:osd.5> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T05:24:19.139 INFO:tasks.cephadm:Deploying osd.6 on smithi110 with /dev/vg_nvme/lv_2... 2024-09-13T05:24:19.139 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T05:24:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:19 smithi110 ceph-mon[25804]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:19 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:19 smithi110 ceph-mon[25804]: from='osd.5 [v2:172.21.15.110:6808/846480745,v1:172.21.15.110:6809/846480745]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T05:24:19.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:19 smithi110 ceph-mon[25804]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T05:24:19.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:19 smithi053 ceph-mon[23721]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:19.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:19 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:19.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:19 smithi053 ceph-mon[23721]: from='osd.5 [v2:172.21.15.110:6808/846480745,v1:172.21.15.110:6809/846480745]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T05:24:19.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:19 smithi053 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T05:24:20.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-mon[25804]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T05:24:20.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-mon[25804]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T05:24:20.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:20.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-mon[25804]: from='osd.5 [v2:172.21.15.110:6808/846480745,v1:172.21.15.110:6809/846480745]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:20.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-mon[25804]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:20.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:20.746 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[32986]: 2024-09-13T05:24:20.617+0000 7fd78d900700 -1 osd.5 0 waiting for initial osdmap 2024-09-13T05:24:20.746 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:24:20 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[32986]: 2024-09-13T05:24:20.622+0000 7fd787295700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:24:20.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:20 smithi053 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T05:24:20.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:20 smithi053 ceph-mon[23721]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T05:24:20.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:20 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:20.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:20 smithi053 ceph-mon[23721]: from='osd.5 [v2:172.21.15.110:6808/846480745,v1:172.21.15.110:6809/846480745]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:20.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:20 smithi053 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:20.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:20 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:21.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:21 smithi110 ceph-mon[25804]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:21.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:21 smithi110 ceph-mon[25804]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:24:21.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:21 smithi110 ceph-mon[25804]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T05:24:21.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:21.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:21 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:21.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:21 smithi053 ceph-mon[23721]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:21.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:21 smithi053 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:24:21.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:21 smithi053 ceph-mon[23721]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T05:24:21.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:21.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:21 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:22.130 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:24:22.149 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_2 2024-09-13T05:24:22.814 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:22 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:24:22.814 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:22 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:24:22.814 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:22 smithi110 ceph-mon[25804]: osd.5 [v2:172.21.15.110:6808/846480745,v1:172.21.15.110:6809/846480745] boot 2024-09-13T05:24:22.814 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:22 smithi110 ceph-mon[25804]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T05:24:22.814 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:22 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:22.814 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:22 smithi110 ceph-mon[25804]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T05:24:22.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:22 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:24:22.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:22 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:24:22.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:22 smithi053 ceph-mon[23721]: osd.5 [v2:172.21.15.110:6808/846480745,v1:172.21.15.110:6809/846480745] boot 2024-09-13T05:24:22.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:22 smithi053 ceph-mon[23721]: osdmap e39: 6 total, 6 up, 6 in 2024-09-13T05:24:22.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:22 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:24:22.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:22 smithi053 ceph-mon[23721]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T05:24:23.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:23 smithi110 ceph-mon[25804]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:23.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:23 smithi110 ceph-mon[25804]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T05:24:23.910 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:23 smithi053 ceph-mon[23721]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T05:24:23.911 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:23 smithi053 ceph-mon[23721]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T05:24:25.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:25 smithi110 ceph-mon[25804]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:25.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:25 smithi110 ceph-mon[25804]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T05:24:25.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:25 smithi110 ceph-mon[25804]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T05:24:25.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:25 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:24:25.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:25 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:24:25.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:25 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:25.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:25 smithi053 ceph-mon[23721]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:25.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:25 smithi053 ceph-mon[23721]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T05:24:25.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:25 smithi053 ceph-mon[23721]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T05:24:25.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:25 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:24:25.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:25 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:24:25.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:25 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:26.839 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:24:26.839 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T05:24:26.839 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: Detected new or changed devices on smithi110 2024-09-13T05:24:26.839 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:26.839 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi110 to 7968M 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:26.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:26 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:26.888 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='client.24195 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:24:26.888 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: Detected new or changed devices on smithi110 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi110 to 7968M 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:26.889 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:26 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:27 smithi110 ceph-mon[25804]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:27 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/3238497272' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f372626a-56fc-42ce-b77f-be5758a77062"}]: dispatch 2024-09-13T05:24:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:27 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f372626a-56fc-42ce-b77f-be5758a77062"}]: dispatch 2024-09-13T05:24:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:27 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f372626a-56fc-42ce-b77f-be5758a77062"}]': finished 2024-09-13T05:24:28.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:27 smithi110 ceph-mon[25804]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T05:24:28.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:27 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:27 smithi053 ceph-mon[23721]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:27 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/3238497272' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f372626a-56fc-42ce-b77f-be5758a77062"}]: dispatch 2024-09-13T05:24:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:27 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f372626a-56fc-42ce-b77f-be5758a77062"}]: dispatch 2024-09-13T05:24:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:27 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f372626a-56fc-42ce-b77f-be5758a77062"}]': finished 2024-09-13T05:24:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:27 smithi053 ceph-mon[23721]: osdmap e44: 7 total, 6 up, 7 in 2024-09-13T05:24:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:27 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:29.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:28 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/3876608773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:24:29.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:28 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:29.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:28 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/3876608773' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:24:29.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:28 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:29 smithi110 ceph-mon[25804]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-09-13T05:24:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:29 smithi110 ceph-mon[25804]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T05:24:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:29 smithi110 ceph-mon[25804]: Cluster is now healthy 2024-09-13T05:24:30.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:29 smithi053 ceph-mon[23721]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 109 KiB/s, 0 objects/s recovering 2024-09-13T05:24:30.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:29 smithi053 ceph-mon[23721]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T05:24:30.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:29 smithi053 ceph-mon[23721]: Cluster is now healthy 2024-09-13T05:24:32.111 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:31 smithi110 ceph-mon[25804]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T05:24:32.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:31 smithi053 ceph-mon[23721]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T05:24:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:33 smithi110 ceph-mon[25804]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-13T05:24:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T05:24:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:34.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:33 smithi053 ceph-mon[23721]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-13T05:24:34.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T05:24:34.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:34.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:34 smithi053 ceph-mon[23721]: Deploying daemon osd.6 on smithi110 2024-09-13T05:24:35.105 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:34 smithi110 ceph-mon[25804]: Deploying daemon osd.6 on smithi110 2024-09-13T05:24:36.104 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:35 smithi110 ceph-mon[25804]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T05:24:36.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:35 smithi053 ceph-mon[23721]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T05:24:38.240 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:37 smithi110 ceph-mon[25804]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:24:38.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:37 smithi053 ceph-mon[23721]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:24:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:39 smithi110 ceph-mon[25804]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:39 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:39.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:39 smithi053 ceph-mon[23721]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:39.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:39.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:39 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:41.607 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:41 smithi110 ceph-mon[25804]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:41.993 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 6 on host 'smithi110' 2024-09-13T05:24:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:41 smithi053 ceph-mon[23721]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:42.587 DEBUG:teuthology.orchestra.run.smithi110:osd.6> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T05:24:42.590 INFO:tasks.cephadm:Deploying osd.7 on smithi110 with /dev/vg_nvme/lv_1... 2024-09-13T05:24:42.591 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T05:24:42.858 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:42 smithi110 ceph-mon[25804]: from='osd.6 [v2:172.21.15.110:6816/1802202469,v1:172.21.15.110:6817/1802202469]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T05:24:42.858 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:42 smithi110 ceph-mon[25804]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T05:24:42.859 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:42 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:42.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:42 smithi053 ceph-mon[23721]: from='osd.6 [v2:172.21.15.110:6816/1802202469,v1:172.21.15.110:6817/1802202469]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T05:24:42.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:42 smithi053 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T05:24:42.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:42 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-mon[25804]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-mon[25804]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T05:24:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-mon[25804]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T05:24:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:43.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-mon[25804]: from='osd.6 [v2:172.21.15.110:6816/1802202469,v1:172.21.15.110:6817/1802202469]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:43.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-mon[25804]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:43.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:43.745 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[35939]: 2024-09-13T05:24:43.578+0000 7fe4a30c6700 -1 osd.6 0 waiting for initial osdmap 2024-09-13T05:24:43.745 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:24:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[35939]: 2024-09-13T05:24:43.584+0000 7fe49fa61700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:24:43.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:43 smithi053 ceph-mon[23721]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:43.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:43 smithi053 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T05:24:43.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:43 smithi053 ceph-mon[23721]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T05:24:43.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:43.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:43 smithi053 ceph-mon[23721]: from='osd.6 [v2:172.21.15.110:6816/1802202469,v1:172.21.15.110:6817/1802202469]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:43.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:43 smithi053 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:24:43.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:44.619 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:24:44.619 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T05:24:44.619 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:44.619 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:44.619 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:44.620 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: osd.6 [v2:172.21.15.110:6816/1802202469,v1:172.21.15.110:6817/1802202469] boot 2024-09-13T05:24:44.620 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T05:24:44.620 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:44 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:44.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:24:44.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T05:24:44.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:44.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:44.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:44.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: osd.6 [v2:172.21.15.110:6816/1802202469,v1:172.21.15.110:6817/1802202469] boot 2024-09-13T05:24:44.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: osdmap e47: 7 total, 7 up, 7 in 2024-09-13T05:24:44.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:44 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:24:45.478 INFO:teuthology.orchestra.run.smithi110.stdout: 2024-09-13T05:24:45.498 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi110:vg_nvme/lv_1 2024-09-13T05:24:45.784 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:45 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:24:45.785 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:45 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:24:45.785 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:45 smithi110 ceph-mon[25804]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:45.785 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:45 smithi110 ceph-mon[25804]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T05:24:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:45 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:24:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:45 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:24:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:45 smithi053 ceph-mon[23721]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T05:24:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:45 smithi053 ceph-mon[23721]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T05:24:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:47 smithi110 ceph-mon[25804]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:24:47.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:47 smithi110 ceph-mon[25804]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T05:24:47.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:47 smithi053 ceph-mon[23721]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:24:47.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:47 smithi053 ceph-mon[23721]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T05:24:48.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:48 smithi110 ceph-mon[25804]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T05:24:48.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:48 smithi053 ceph-mon[23721]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T05:24:49.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T05:24:49.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T05:24:49.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='client.24213 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:24:49.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:24:49.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:24:49.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:49.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:49.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: pgmap v153: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T05:24:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T05:24:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='client.24213 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi110:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:24:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T05:24:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:24:49.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:24:50.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:50 smithi110 ceph-mon[25804]: Detected new or changed devices on smithi110 2024-09-13T05:24:50.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:50 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi110 to 5312M 2024-09-13T05:24:50.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:50 smithi053 ceph-mon[23721]: Detected new or changed devices on smithi110 2024-09-13T05:24:50.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:50 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi110 to 5312M 2024-09-13T05:24:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:51 smithi110 ceph-mon[25804]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-09-13T05:24:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:51 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/2503668907' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fa385f8-ef89-42c9-b405-36c8fb56ab4d"}]: dispatch 2024-09-13T05:24:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:51 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fa385f8-ef89-42c9-b405-36c8fb56ab4d"}]: dispatch 2024-09-13T05:24:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:51 smithi110 ceph-mon[25804]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fa385f8-ef89-42c9-b405-36c8fb56ab4d"}]': finished 2024-09-13T05:24:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:51 smithi110 ceph-mon[25804]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T05:24:51.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:24:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:51 smithi053 ceph-mon[23721]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-09-13T05:24:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:51 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/2503668907' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fa385f8-ef89-42c9-b405-36c8fb56ab4d"}]: dispatch 2024-09-13T05:24:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:51 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3fa385f8-ef89-42c9-b405-36c8fb56ab4d"}]: dispatch 2024-09-13T05:24:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:51 smithi053 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3fa385f8-ef89-42c9-b405-36c8fb56ab4d"}]': finished 2024-09-13T05:24:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:51 smithi053 ceph-mon[23721]: osdmap e52: 8 total, 7 up, 8 in 2024-09-13T05:24:51.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:24:52.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:52 smithi053 ceph-mon[23721]: from='client.? 172.21.15.110:0/3388064246' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:24:52.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:52 smithi110 ceph-mon[25804]: from='client.? 172.21.15.110:0/3388064246' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T05:24:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:53 smithi110 ceph-mon[25804]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-13T05:24:53.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:53 smithi053 ceph-mon[23721]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-13T05:24:55.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:55 smithi110 ceph-mon[25804]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:24:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:55 smithi053 ceph-mon[23721]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:24:57.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:57 smithi110 ceph-mon[25804]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:24:57.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:57 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T05:24:57.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:57 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:57.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:57 smithi053 ceph-mon[23721]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:24:57.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:57 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T05:24:57.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:57 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:24:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:58 smithi110 ceph-mon[25804]: Deploying daemon osd.7 on smithi110 2024-09-13T05:24:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:24:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:24:58.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:58 smithi053 ceph-mon[23721]: Deploying daemon osd.7 on smithi110 2024-09-13T05:24:58.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:24:58.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:24:59.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:24:59 smithi110 ceph-mon[25804]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:24:59.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:24:59 smithi053 ceph-mon[23721]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:01.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:01 smithi110 ceph-mon[25804]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:01.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:01 smithi053 ceph-mon[23721]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:03.686 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:03 smithi110 ceph-mon[25804]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:03.686 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:03 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:03.686 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:03 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:25:03.686 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:03 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:25:03.686 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:03 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:25:03.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:03 smithi053 ceph-mon[23721]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:03.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:03 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:03.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:03 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:25:03.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:03 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:25:03.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:03 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:25:05.027 INFO:teuthology.orchestra.run.smithi110.stdout:Created osd(s) 7 on host 'smithi110' 2024-09-13T05:25:05.596 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:05 smithi110 ceph-mon[25804]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:05.597 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:05 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:05.619 DEBUG:teuthology.orchestra.run.smithi110:osd.7> sudo journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T05:25:05.621 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T05:25:05.621 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T05:25:05.939 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:05 smithi053 ceph-mon[23721]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:05.939 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:05 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:06.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:06 smithi110 ceph-mon[25804]: from='osd.7 [v2:172.21.15.110:6824/2902762001,v1:172.21.15.110:6825/2902762001]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T05:25:06.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:06 smithi110 ceph-mon[25804]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T05:25:06.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:06 smithi053 ceph-mon[23721]: from='osd.7 [v2:172.21.15.110:6824/2902762001,v1:172.21.15.110:6825/2902762001]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T05:25:06.840 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:06 smithi053 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T05:25:07.639 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:25:07.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:07 smithi053 ceph-mon[23721]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:07.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:07 smithi053 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T05:25:07.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:07 smithi053 ceph-mon[23721]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T05:25:07.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:07.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:07 smithi053 ceph-mon[23721]: from='osd.7 [v2:172.21.15.110:6824/2902762001,v1:172.21.15.110:6825/2902762001]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:25:07.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:07 smithi053 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:25:07.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:07 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-mon[25804]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-mon[25804]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T05:25:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-mon[25804]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T05:25:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-mon[25804]: from='osd.7 [v2:172.21.15.110:6824/2902762001,v1:172.21.15.110:6825/2902762001]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:25:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-mon[25804]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]: dispatch 2024-09-13T05:25:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:07.995 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[38983]: 2024-09-13T05:25:07.550+0000 7fb70158b700 -1 osd.7 0 waiting for initial osdmap 2024-09-13T05:25:07.995 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:25:07 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[38983]: 2024-09-13T05:25:07.557+0000 7fb6f9f1e700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:25:08.322 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1726205084,"num_in_osds":8,"osd_in_since":1726205091,"num_remapped_pgs":0} 2024-09-13T05:25:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:08 smithi110 ceph-mon[25804]: purged_snaps scrub starts 2024-09-13T05:25:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:08 smithi110 ceph-mon[25804]: purged_snaps scrub ok 2024-09-13T05:25:08.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:08 smithi110 ceph-mon[25804]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:25:08.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:08 smithi110 ceph-mon[25804]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T05:25:08.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:08 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:08.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:08 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:08.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:08 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1750063241' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T05:25:08.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:08 smithi053 ceph-mon[23721]: purged_snaps scrub starts 2024-09-13T05:25:08.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:08 smithi053 ceph-mon[23721]: purged_snaps scrub ok 2024-09-13T05:25:08.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:08 smithi053 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]}]': finished 2024-09-13T05:25:08.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:08 smithi053 ceph-mon[23721]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T05:25:08.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:08 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:08.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:08 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:08.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:08 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1750063241' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T05:25:09.324 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T05:25:09.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:09 smithi053 ceph-mon[23721]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:09.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:09 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:09.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:09 smithi053 ceph-mon[23721]: osd.7 [v2:172.21.15.110:6824/2902762001,v1:172.21.15.110:6825/2902762001] boot 2024-09-13T05:25:09.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:09 smithi053 ceph-mon[23721]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T05:25:09.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:09 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:09.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:09 smithi053 ceph-mon[23721]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T05:25:09.853 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:09 smithi110 ceph-mon[25804]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T05:25:09.854 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:09 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:09.854 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:09 smithi110 ceph-mon[25804]: osd.7 [v2:172.21.15.110:6824/2902762001,v1:172.21.15.110:6825/2902762001] boot 2024-09-13T05:25:09.854 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:09 smithi110 ceph-mon[25804]: osdmap e55: 8 total, 8 up, 8 in 2024-09-13T05:25:09.854 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:09 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:25:09.854 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:09 smithi110 ceph-mon[25804]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: Detected new or changed devices on smithi110 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi110 to 3984M 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:10.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:25:10.999 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:25:10.999 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:25:10.999 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:10.999 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:10 smithi053 ceph-mon[23721]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T05:25:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: Detected new or changed devices on smithi110 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi110 to 3984M 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:25:11.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:25:11.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:10 smithi110 ceph-mon[25804]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T05:25:11.408 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:25:12.113 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:11 smithi053 ceph-mon[23721]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:12.113 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:11 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2297926727' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T05:25:12.113 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:11 smithi053 ceph-mon[23721]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T05:25:12.138 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1726205108,"num_in_osds":8,"osd_in_since":1726205091,"num_remapped_pgs":1} 2024-09-13T05:25:12.138 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T05:25:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:11 smithi110 ceph-mon[25804]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:11 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2297926727' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T05:25:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:11 smithi110 ceph-mon[25804]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T05:25:12.355 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:13.795 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:13 smithi053 ceph-mon[23721]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:13.796 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:13 smithi053 ceph-mon[23721]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T05:25:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:13 smithi110 ceph-mon[25804]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:13 smithi110 ceph-mon[25804]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T05:25:14.214 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:25:14.214 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":59,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","created":"2024-09-13T05:19:09.865007+0000","modified":"2024-09-13T05:25:12.542317+0000","last_up_change":"2024-09-13T05:25:08.548218+0000","last_in_change":"2024-09-13T05:24:51.282399+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T05:23:08.807365+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"11bdd334-df82-49e9-86bc-e0da79b90761","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6802","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6803","nonce":1675242571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6804","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6805","nonce":1675242571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6808","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6809","nonce":1675242571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6806","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6807","nonce":1675242571}]},"public_addr":"172.21.15.53:6803/1675242571","cluster_addr":"172.21.15.53:6805/1675242571","heartbeat_back_addr":"172.21.15.53:6809/1675242571","heartbeat_front_addr":"172.21.15.53:6807/1675242571","state":["exists","up"]},{"osd":1,"uuid":"4c4006a9-cb6a-4f0d-a328-7d56c363afe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6810","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6811","nonce":3213587156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6812","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6813","nonce":3213587156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6816","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6817","nonce":3213587156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6814","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6815","nonce":3213587156}]},"public_addr":"172.21.15.53:6811/3213587156","cluster_addr":"172.21.15.53:6813/3213587156","heartbeat_back_addr":"172.21.15.53:6817/3213587156","heartbeat_front_addr":"172.21.15.53:6815/3213587156","state":["exists","up"]},{"osd":2,"uuid":"beefd695-6c5d-419d-bf38-acde87bfe835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6818","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6819","nonce":366341317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6820","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6821","nonce":366341317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6824","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6825","nonce":366341317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6822","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6823","nonce":366341317}]},"public_addr":"172.21.15.53:6819/366341317","cluster_addr":"172.21.15.53:6821/366341317","heartbeat_back_addr":"172.21.15.53:6825/366341317","heartbeat_front_addr":"172.21.15.53:6823/366341317","state":["exists","up"]},{"osd":3,"uuid":"7dd6a140-c882-467e-9c50-fe9f9a1f57ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6826","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6827","nonce":3452758796}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6828","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6829","nonce":3452758796}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6832","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6833","nonce":3452758796}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6830","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6831","nonce":3452758796}]},"public_addr":"172.21.15.53:6827/3452758796","cluster_addr":"172.21.15.53:6829/3452758796","heartbeat_back_addr":"172.21.15.53:6833/3452758796","heartbeat_front_addr":"172.21.15.53:6831/3452758796","state":["exists","up"]},{"osd":4,"uuid":"988194ce-8aad-4b6c-8269-1c1a6fdabb0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6801","nonce":3538242982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6803","nonce":3538242982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6807","nonce":3538242982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6805","nonce":3538242982}]},"public_addr":"172.21.15.110:6801/3538242982","cluster_addr":"172.21.15.110:6803/3538242982","heartbeat_back_addr":"172.21.15.110:6807/3538242982","heartbeat_front_addr":"172.21.15.110:6805/3538242982","state":["exists","up"]},{"osd":5,"uuid":"7e15755a-7760-4cbd-ac69-400b6de3f247","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6809","nonce":846480745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6811","nonce":846480745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6815","nonce":846480745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6813","nonce":846480745}]},"public_addr":"172.21.15.110:6809/846480745","cluster_addr":"172.21.15.110:6811/846480745","heartbeat_back_addr":"172.21.15.110:6815/846480745","heartbeat_front_addr":"172.21.15.110:6813/846480745","state":["exists","up"]},{"osd":6,"uuid":"f372626a-56fc-42ce-b77f-be5758a77062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6817","nonce":1802202469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6819","nonce":1802202469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6823","nonce":1802202469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6821","nonce":1802202469}]},"public_addr":"172.21.15.110:6817/1802202469","cluster_addr":"172.21.15.110:6819/1802202469","heartbeat_back_addr":"172.21.15.110:6823/1802202469","heartbeat_front_addr":"172.21.15.110:6821/1802202469","state":["exists","up"]},{"osd":7,"uuid":"3fa385f8-ef89-42c9-b405-36c8fb56ab4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6825","nonce":2902762001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6827","nonce":2902762001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6831","nonce":2902762001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6829","nonce":2902762001}]},"public_addr":"172.21.15.110:6825/2902762001","cluster_addr":"172.21.15.110:6827/2902762001","heartbeat_back_addr":"172.21.15.110:6831/2902762001","heartbeat_front_addr":"172.21.15.110:6829/2902762001","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:22:10.691520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:22:37.015398+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:06.133136+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:34.568236+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:55.877966+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:24:19.623566+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:24:42.660774+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:25:06.519888+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.53:0/3595623270":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/1800716634":"2024-09-14T05:20:58.251789+0000","172.21.15.53:6800/2353383206":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/2774234647":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/4062231106":"2024-09-14T05:20:07.057856+0000","172.21.15.53:6801/1206183288":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/4029356077":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/494195240":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/758940679":"2024-09-14T05:19:36.766045+0000","172.21.15.53:6801/2353383206":"2024-09-14T05:20:58.251789+0000","172.21.15.53:6800/445573966":"2024-09-14T05:19:36.766045+0000","172.21.15.53:0/3919975874":"2024-09-14T05:19:36.766045+0000","172.21.15.53:6800/1206183288":"2024-09-14T05:20:07.057856+0000","172.21.15.53:6801/445573966":"2024-09-14T05:19:36.766045+0000","172.21.15.53:0/833725675":"2024-09-14T05:19:36.766045+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T05:25:14.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:14 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1263004891' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T05:25:14.860 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T05:23:08.807365+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T05:25:14.861 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T05:25:14.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:14 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1263004891' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T05:25:15.076 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:15 smithi110 ceph-mon[25804]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:15.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:15 smithi053 ceph-mon[23721]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:16.899 INFO:teuthology.orchestra.run.smithi053.stdout:pg_num: 1 2024-09-13T05:25:17.591 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T05:25:17.591 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T05:25:17.809 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:17 smithi110 ceph-mon[25804]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T05:25:17.810 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:17 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/946513533' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T05:25:17.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:17 smithi053 ceph-mon[23721]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T05:25:17.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:17 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/946513533' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T05:25:19.676 INFO:teuthology.orchestra.run.smithi053.stdout:[client.0] 2024-09-13T05:25:19.677 INFO:teuthology.orchestra.run.smithi053.stdout: key = AQC/zONmDeshKBAA3J4SJVMRP2QV3tuwm9zRbQ== 2024-09-13T05:25:19.970 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:19 smithi053 ceph-mon[23721]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-13T05:25:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:19 smithi110 ceph-mon[25804]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-09-13T05:25:20.325 DEBUG:teuthology.orchestra.run.smithi053:> set -ex 2024-09-13T05:25:20.326 DEBUG:teuthology.orchestra.run.smithi053:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T05:25:20.326 DEBUG:teuthology.orchestra.run.smithi053:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T05:25:20.367 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-13T05:25:20.625 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:20.871 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:20 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1423894876' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T05:25:20.871 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:20 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1423894876' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T05:25:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:20 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1423894876' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T05:25:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:20 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1423894876' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T05:25:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:21 smithi110 ceph-mon[25804]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T05:25:21.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:21 smithi053 ceph-mon[23721]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-09-13T05:25:22.437 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:25:22.693 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:22 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2242822784' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T05:25:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:22 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2242822784' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T05:25:23.042 INFO:teuthology.orchestra.run.smithi053.stdout:[{"version":20,"timestamp":"2024-09-13T05:25:09.890152+0000","name":"","changes":[{"name":"osd/host:smithi110/osd_memory_target","previous_value":"5570043494","new_value":"4177532620"}]}] 2024-09-13T05:25:23.042 INFO:tasks.ceph_manager:config epoch is 20 2024-09-13T05:25:23.042 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T05:25:23.042 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T05:25:23.043 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T05:25:23.260 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:23 smithi110 ceph-mon[25804]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:25:23.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:23 smithi053 ceph-mon[23721]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-13T05:25:25.187 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:25:25.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:25 smithi053 ceph-mon[23721]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-13T05:25:25.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:25 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1092197380' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T05:25:25.920 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi053.ygzjut","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6800","nonce":847951478},{"type":"v1","addr":"172.21.15.53:6801","nonce":847951478}]},"active_addr":"172.21.15.53:6801/847951478","active_change":"2024-09-13T05:20:58.252071+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi110.rivldx","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.53:8443/","prometheus":"http://172.21.15.53:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.53:0","nonce":1343322177}]},{"addrvec":[{"type":"v2","addr":"172.21.15.53:0","nonce":2909467733}]},{"addrvec":[{"type":"v2","addr":"172.21.15.53:0","nonce":2740021115}]},{"addrvec":[{"type":"v2","addr":"172.21.15.53:0","nonce":1493870336}]}]}} 2024-09-13T05:25:25.924 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T05:25:25.924 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T05:25:25.924 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T05:25:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:25 smithi110 ceph-mon[25804]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-13T05:25:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:25 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1092197380' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T05:25:26.150 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:27.737 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:25:27.737 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":59,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","created":"2024-09-13T05:19:09.865007+0000","modified":"2024-09-13T05:25:12.542317+0000","last_up_change":"2024-09-13T05:25:08.548218+0000","last_in_change":"2024-09-13T05:24:51.282399+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T05:23:08.807365+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"11bdd334-df82-49e9-86bc-e0da79b90761","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6802","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6803","nonce":1675242571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6804","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6805","nonce":1675242571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6808","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6809","nonce":1675242571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6806","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6807","nonce":1675242571}]},"public_addr":"172.21.15.53:6803/1675242571","cluster_addr":"172.21.15.53:6805/1675242571","heartbeat_back_addr":"172.21.15.53:6809/1675242571","heartbeat_front_addr":"172.21.15.53:6807/1675242571","state":["exists","up"]},{"osd":1,"uuid":"4c4006a9-cb6a-4f0d-a328-7d56c363afe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6810","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6811","nonce":3213587156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6812","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6813","nonce":3213587156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6816","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6817","nonce":3213587156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6814","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6815","nonce":3213587156}]},"public_addr":"172.21.15.53:6811/3213587156","cluster_addr":"172.21.15.53:6813/3213587156","heartbeat_back_addr":"172.21.15.53:6817/3213587156","heartbeat_front_addr":"172.21.15.53:6815/3213587156","state":["exists","up"]},{"osd":2,"uuid":"beefd695-6c5d-419d-bf38-acde87bfe835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6818","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6819","nonce":366341317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6820","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6821","nonce":366341317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6824","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6825","nonce":366341317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6822","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6823","nonce":366341317}]},"public_addr":"172.21.15.53:6819/366341317","cluster_addr":"172.21.15.53:6821/366341317","heartbeat_back_addr":"172.21.15.53:6825/366341317","heartbeat_front_addr":"172.21.15.53:6823/366341317","state":["exists","up"]},{"osd":3,"uuid":"7dd6a140-c882-467e-9c50-fe9f9a1f57ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6826","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6827","nonce":3452758796}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6828","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6829","nonce":3452758796}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6832","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6833","nonce":3452758796}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6830","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6831","nonce":3452758796}]},"public_addr":"172.21.15.53:6827/3452758796","cluster_addr":"172.21.15.53:6829/3452758796","heartbeat_back_addr":"172.21.15.53:6833/3452758796","heartbeat_front_addr":"172.21.15.53:6831/3452758796","state":["exists","up"]},{"osd":4,"uuid":"988194ce-8aad-4b6c-8269-1c1a6fdabb0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6801","nonce":3538242982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6803","nonce":3538242982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6807","nonce":3538242982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6805","nonce":3538242982}]},"public_addr":"172.21.15.110:6801/3538242982","cluster_addr":"172.21.15.110:6803/3538242982","heartbeat_back_addr":"172.21.15.110:6807/3538242982","heartbeat_front_addr":"172.21.15.110:6805/3538242982","state":["exists","up"]},{"osd":5,"uuid":"7e15755a-7760-4cbd-ac69-400b6de3f247","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6809","nonce":846480745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6811","nonce":846480745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6815","nonce":846480745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6813","nonce":846480745}]},"public_addr":"172.21.15.110:6809/846480745","cluster_addr":"172.21.15.110:6811/846480745","heartbeat_back_addr":"172.21.15.110:6815/846480745","heartbeat_front_addr":"172.21.15.110:6813/846480745","state":["exists","up"]},{"osd":6,"uuid":"f372626a-56fc-42ce-b77f-be5758a77062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6817","nonce":1802202469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6819","nonce":1802202469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6823","nonce":1802202469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6821","nonce":1802202469}]},"public_addr":"172.21.15.110:6817/1802202469","cluster_addr":"172.21.15.110:6819/1802202469","heartbeat_back_addr":"172.21.15.110:6823/1802202469","heartbeat_front_addr":"172.21.15.110:6821/1802202469","state":["exists","up"]},{"osd":7,"uuid":"3fa385f8-ef89-42c9-b405-36c8fb56ab4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6825","nonce":2902762001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6827","nonce":2902762001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6831","nonce":2902762001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6829","nonce":2902762001}]},"public_addr":"172.21.15.110:6825/2902762001","cluster_addr":"172.21.15.110:6827/2902762001","heartbeat_back_addr":"172.21.15.110:6831/2902762001","heartbeat_front_addr":"172.21.15.110:6829/2902762001","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:22:10.691520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:22:37.015398+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:06.133136+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:34.568236+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:55.877966+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:24:19.623566+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:24:42.660774+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:25:06.519888+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.53:0/3595623270":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/1800716634":"2024-09-14T05:20:58.251789+0000","172.21.15.53:6800/2353383206":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/2774234647":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/4062231106":"2024-09-14T05:20:07.057856+0000","172.21.15.53:6801/1206183288":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/4029356077":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/494195240":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/758940679":"2024-09-14T05:19:36.766045+0000","172.21.15.53:6801/2353383206":"2024-09-14T05:20:58.251789+0000","172.21.15.53:6800/445573966":"2024-09-14T05:19:36.766045+0000","172.21.15.53:0/3919975874":"2024-09-14T05:19:36.766045+0000","172.21.15.53:6800/1206183288":"2024-09-14T05:20:07.057856+0000","172.21.15.53:6801/445573966":"2024-09-14T05:19:36.766045+0000","172.21.15.53:0/833725675":"2024-09-14T05:19:36.766045+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T05:25:27.737 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:27 smithi053 ceph-mon[23721]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-13T05:25:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:27 smithi110 ceph-mon[25804]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-13T05:25:28.410 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T05:25:28.411 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T05:25:28.626 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:28.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:28 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3609606285' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T05:25:28.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:28 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3609606285' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T05:25:29.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:29 smithi053 ceph-mon[23721]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:29 smithi110 ceph-mon[25804]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:30.481 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:25:30.482 INFO:teuthology.orchestra.run.smithi053.stdout:{"epoch":59,"fsid":"9f95c28e-718f-11ef-bceb-c7b262605968","created":"2024-09-13T05:19:09.865007+0000","modified":"2024-09-13T05:25:12.542317+0000","last_up_change":"2024-09-13T05:25:08.548218+0000","last_in_change":"2024-09-13T05:24:51.282399+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T05:23:08.807365+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"11bdd334-df82-49e9-86bc-e0da79b90761","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6802","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6803","nonce":1675242571}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6804","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6805","nonce":1675242571}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6808","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6809","nonce":1675242571}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6806","nonce":1675242571},{"type":"v1","addr":"172.21.15.53:6807","nonce":1675242571}]},"public_addr":"172.21.15.53:6803/1675242571","cluster_addr":"172.21.15.53:6805/1675242571","heartbeat_back_addr":"172.21.15.53:6809/1675242571","heartbeat_front_addr":"172.21.15.53:6807/1675242571","state":["exists","up"]},{"osd":1,"uuid":"4c4006a9-cb6a-4f0d-a328-7d56c363afe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6810","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6811","nonce":3213587156}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6812","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6813","nonce":3213587156}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6816","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6817","nonce":3213587156}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6814","nonce":3213587156},{"type":"v1","addr":"172.21.15.53:6815","nonce":3213587156}]},"public_addr":"172.21.15.53:6811/3213587156","cluster_addr":"172.21.15.53:6813/3213587156","heartbeat_back_addr":"172.21.15.53:6817/3213587156","heartbeat_front_addr":"172.21.15.53:6815/3213587156","state":["exists","up"]},{"osd":2,"uuid":"beefd695-6c5d-419d-bf38-acde87bfe835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6818","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6819","nonce":366341317}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6820","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6821","nonce":366341317}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6824","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6825","nonce":366341317}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6822","nonce":366341317},{"type":"v1","addr":"172.21.15.53:6823","nonce":366341317}]},"public_addr":"172.21.15.53:6819/366341317","cluster_addr":"172.21.15.53:6821/366341317","heartbeat_back_addr":"172.21.15.53:6825/366341317","heartbeat_front_addr":"172.21.15.53:6823/366341317","state":["exists","up"]},{"osd":3,"uuid":"7dd6a140-c882-467e-9c50-fe9f9a1f57ae","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6826","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6827","nonce":3452758796}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6828","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6829","nonce":3452758796}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6832","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6833","nonce":3452758796}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.53:6830","nonce":3452758796},{"type":"v1","addr":"172.21.15.53:6831","nonce":3452758796}]},"public_addr":"172.21.15.53:6827/3452758796","cluster_addr":"172.21.15.53:6829/3452758796","heartbeat_back_addr":"172.21.15.53:6833/3452758796","heartbeat_front_addr":"172.21.15.53:6831/3452758796","state":["exists","up"]},{"osd":4,"uuid":"988194ce-8aad-4b6c-8269-1c1a6fdabb0f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6800","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6801","nonce":3538242982}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6802","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6803","nonce":3538242982}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6806","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6807","nonce":3538242982}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6804","nonce":3538242982},{"type":"v1","addr":"172.21.15.110:6805","nonce":3538242982}]},"public_addr":"172.21.15.110:6801/3538242982","cluster_addr":"172.21.15.110:6803/3538242982","heartbeat_back_addr":"172.21.15.110:6807/3538242982","heartbeat_front_addr":"172.21.15.110:6805/3538242982","state":["exists","up"]},{"osd":5,"uuid":"7e15755a-7760-4cbd-ac69-400b6de3f247","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6808","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6809","nonce":846480745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6810","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6811","nonce":846480745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6814","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6815","nonce":846480745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6812","nonce":846480745},{"type":"v1","addr":"172.21.15.110:6813","nonce":846480745}]},"public_addr":"172.21.15.110:6809/846480745","cluster_addr":"172.21.15.110:6811/846480745","heartbeat_back_addr":"172.21.15.110:6815/846480745","heartbeat_front_addr":"172.21.15.110:6813/846480745","state":["exists","up"]},{"osd":6,"uuid":"f372626a-56fc-42ce-b77f-be5758a77062","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6816","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6817","nonce":1802202469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6818","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6819","nonce":1802202469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6822","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6823","nonce":1802202469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6820","nonce":1802202469},{"type":"v1","addr":"172.21.15.110:6821","nonce":1802202469}]},"public_addr":"172.21.15.110:6817/1802202469","cluster_addr":"172.21.15.110:6819/1802202469","heartbeat_back_addr":"172.21.15.110:6823/1802202469","heartbeat_front_addr":"172.21.15.110:6821/1802202469","state":["exists","up"]},{"osd":7,"uuid":"3fa385f8-ef89-42c9-b405-36c8fb56ab4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6824","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6825","nonce":2902762001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6826","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6827","nonce":2902762001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6830","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6831","nonce":2902762001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.110:6828","nonce":2902762001},{"type":"v1","addr":"172.21.15.110:6829","nonce":2902762001}]},"public_addr":"172.21.15.110:6825/2902762001","cluster_addr":"172.21.15.110:6827/2902762001","heartbeat_back_addr":"172.21.15.110:6831/2902762001","heartbeat_front_addr":"172.21.15.110:6829/2902762001","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:22:10.691520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:22:37.015398+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:06.133136+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:34.568236+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:23:55.877966+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:24:19.623566+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:24:42.660774+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T05:25:06.519888+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.53:0/3595623270":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/1800716634":"2024-09-14T05:20:58.251789+0000","172.21.15.53:6800/2353383206":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/2774234647":"2024-09-14T05:20:58.251789+0000","172.21.15.53:0/4062231106":"2024-09-14T05:20:07.057856+0000","172.21.15.53:6801/1206183288":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/4029356077":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/494195240":"2024-09-14T05:20:07.057856+0000","172.21.15.53:0/758940679":"2024-09-14T05:19:36.766045+0000","172.21.15.53:6801/2353383206":"2024-09-14T05:20:58.251789+0000","172.21.15.53:6800/445573966":"2024-09-14T05:19:36.766045+0000","172.21.15.53:0/3919975874":"2024-09-14T05:19:36.766045+0000","172.21.15.53:6800/1206183288":"2024-09-14T05:20:07.057856+0000","172.21.15.53:6801/445573966":"2024-09-14T05:19:36.766045+0000","172.21.15.53:0/833725675":"2024-09-14T05:19:36.766045+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T05:25:30.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:30 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1836261756' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T05:25:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:30 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1836261756' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T05:25:31.090 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T05:25:31.091 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T05:25:31.091 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T05:25:31.092 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T05:25:31.092 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T05:25:31.093 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T05:25:31.093 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T05:25:31.094 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T05:25:31.496 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.500 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.501 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.501 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.506 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.512 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.514 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.529 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:31.910 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:31 smithi053 ceph-mon[23721]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:31 smithi110 ceph-mon[25804]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:33.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:33 smithi053 ceph-mon[23721]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:33 smithi110 ceph-mon[25804]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:35 smithi110 ceph-mon[25804]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:35.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:35 smithi053 ceph-mon[23721]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:37.992 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:37 smithi053 ceph-mon[23721]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:37 smithi110 ceph-mon[25804]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:39 smithi110 ceph-mon[25804]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:39.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:39 smithi053 ceph-mon[23721]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:41.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:41 smithi053 ceph-mon[23721]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:41 smithi110 ceph-mon[25804]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:42.016 INFO:teuthology.orchestra.run.smithi053.stdout:236223201289 2024-09-13T05:25:42.017 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T05:25:43.242 INFO:teuthology.orchestra.run.smithi053.stdout:201863462926 2024-09-13T05:25:43.242 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T05:25:43.837 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:43 smithi053 ceph-mon[23721]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:43.859 INFO:teuthology.orchestra.run.smithi053.stdout:38654705708 2024-09-13T05:25:43.859 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T05:25:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:43 smithi110 ceph-mon[25804]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:44.472 INFO:teuthology.orchestra.run.smithi053.stdout:133143986200 2024-09-13T05:25:44.472 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T05:25:44.918 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:45.796 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:45 smithi053 ceph-mon[23721]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:45 smithi110 ceph-mon[25804]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:46.409 INFO:teuthology.orchestra.run.smithi053.stdout:60129542185 2024-09-13T05:25:46.409 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T05:25:47.315 INFO:teuthology.orchestra.run.smithi053.stdout:111669149725 2024-09-13T05:25:47.316 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T05:25:47.533 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:47.533 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:47.534 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:47.912 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:47 smithi053 ceph-mon[23721]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:47.936 INFO:teuthology.orchestra.run.smithi053.stdout:167503724564 2024-09-13T05:25:47.937 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T05:25:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:47 smithi110 ceph-mon[25804]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:48.573 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:48.573 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:48.574 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:49.328 INFO:teuthology.orchestra.run.smithi053.stdout:81604378658 2024-09-13T05:25:49.329 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T05:25:49.716 INFO:teuthology.orchestra.run.smithi053.stdout:236223201291 2024-09-13T05:25:49.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:49 smithi053 ceph-mon[23721]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:49.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:49 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3937833533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T05:25:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:49 smithi110 ceph-mon[25804]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:49 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3937833533' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T05:25:51.203 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201291 for osd.7 2024-09-13T05:25:51.204 DEBUG:teuthology.parallel:result is None 2024-09-13T05:25:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:51 smithi110 ceph-mon[25804]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:51 smithi053 ceph-mon[23721]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:52.123 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:25:53.922 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:53 smithi053 ceph-mon[23721]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:53 smithi110 ceph-mon[25804]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:55.344 INFO:teuthology.orchestra.run.smithi053.stdout:38654705711 2024-09-13T05:25:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:55 smithi110 ceph-mon[25804]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:55 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/882361886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T05:25:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:55 smithi053 ceph-mon[23721]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:55 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/882361886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T05:25:57.540 INFO:teuthology.orchestra.run.smithi053.stdout:167503724566 2024-09-13T05:25:57.989 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:57 smithi053 ceph-mon[23721]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:57.989 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:57 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3023598753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T05:25:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:57 smithi110 ceph-mon[25804]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:57 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3023598753' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T05:25:58.116 INFO:teuthology.orchestra.run.smithi053.stdout:81604378660 2024-09-13T05:25:58.843 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:58 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2529411593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T05:25:58.843 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:25:58.844 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:25:58.868 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705711 for osd.0 2024-09-13T05:25:58.868 DEBUG:teuthology.parallel:result is None 2024-09-13T05:25:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:58 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2529411593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T05:25:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:25:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:25:59.983 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:25:59 smithi053 ceph-mon[23721]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:25:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:25:59 smithi110 ceph-mon[25804]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:00.172 INFO:teuthology.orchestra.run.smithi053.stdout:201863462929 2024-09-13T05:26:00.504 INFO:teuthology.orchestra.run.smithi053.stdout:133143986203 2024-09-13T05:26:00.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:00 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/915725802' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T05:26:00.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:00 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1585897530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T05:26:00.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:00 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/915725802' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T05:26:00.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:00 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1585897530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T05:26:01.062 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378660 for osd.2 2024-09-13T05:26:01.062 DEBUG:teuthology.parallel:result is None 2024-09-13T05:26:01.724 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986203 for osd.4 2024-09-13T05:26:01.725 DEBUG:teuthology.parallel:result is None 2024-09-13T05:26:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:01 smithi110 ceph-mon[25804]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:01.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:01 smithi053 ceph-mon[23721]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:02.070 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724564 got 167503724566 for osd.5 2024-09-13T05:26:02.071 DEBUG:teuthology.parallel:result is None 2024-09-13T05:26:02.809 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462929 for osd.6 2024-09-13T05:26:02.809 DEBUG:teuthology.parallel:result is None 2024-09-13T05:26:02.889 INFO:teuthology.orchestra.run.smithi053.stdout:60129542188 2024-09-13T05:26:03.401 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542185 got 60129542188 for osd.1 2024-09-13T05:26:03.402 DEBUG:teuthology.parallel:result is None 2024-09-13T05:26:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:03 smithi053 ceph-mon[23721]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:03 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3365973266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T05:26:04.021 INFO:teuthology.orchestra.run.smithi053.stdout:111669149729 2024-09-13T05:26:04.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:03 smithi110 ceph-mon[25804]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:04.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:03 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3365973266' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T05:26:04.589 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149729 for osd.3 2024-09-13T05:26:04.590 DEBUG:teuthology.parallel:result is None 2024-09-13T05:26:04.590 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T05:26:04.590 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T05:26:04.817 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:26:04.932 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:04 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2935212979' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T05:26:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:04 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2935212979' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T05:26:05.897 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:05 smithi053 ceph-mon[23721]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:06.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:05 smithi110 ceph-mon[25804]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:06.581 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:26:06.582 INFO:teuthology.orchestra.run.smithi053.stderr:dumped all 2024-09-13T05:26:07.238 INFO:teuthology.orchestra.run.smithi053.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-13T05:26:06.359607+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48880,"kb_used_data":6320,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912336,"statfs":{"total":767960285184,"available":767910232064,"internally_reserved":0,"allocated":6471680,"data_stored":3548002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002165"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T05:25:12.548458+0000","last_change":"2024-09-13T05:25:12.548458+0000","last_active":"2024-09-13T05:25:12.548458+0000","last_peered":"2024-09-13T05:25:12.548458+0000","last_clean":"2024-09-13T05:25:12.548458+0000","last_became_active":"2024-09-13T05:25:12.547966+0000","last_became_peered":"2024-09-13T05:25:12.547966+0000","last_unstale":"2024-09-13T05:25:12.548458+0000","last_undegraded":"2024-09-13T05:25:12.548458+0000","last_fullsized":"2024-09-13T05:25:12.548458+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T05:23:08.807859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T05:23:08.807859+0000","last_clean_scrub_stamp":"2024-09-13T05:23:08.807859+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:52.923340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65100000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38700000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.438}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.50800000000000001}]},{"osd":1,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.056,"5min":1.056,"15min":1.056},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.60799999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58899999999999997}]},{"osd":3,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.68400000000000005}]},{"osd":4,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.74399999999999999}]},{"osd":5,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.72099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81000000000000005}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.36599999999999999,"15min":0.35199999999999998},"min":{"1min":0.249,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.438},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.34999999999999998},"min":{"1min":0.27000000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.34899999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:14 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.39800000000000002,"15min":0.38900000000000001},"min":{"1min":0.28399999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38,"15min":0.372},"min":{"1min":0.32400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.42399999999999999}]},{"osd":3,"last update":"Fri Sep 13 05:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.441,"15min":0.436},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.45600000000000002}]},{"osd":4,"last update":"Fri Sep 13 05:25:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.61299999999999999}]},{"osd":5,"last update":"Fri Sep 13 05:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.54000000000000004}]},{"osd":6,"last update":"Fri Sep 13 05:25:46 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.59299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 05:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32000000000000001},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.315},"min":{"1min":0.248,"5min":0.22,"15min":0.22},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.502}]},{"osd":2,"last update":"Fri Sep 13 05:25:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.27700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.438,"5min":0.39700000000000002,"15min":0.39000000000000001},"min":{"1min":0.30499999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.42899999999999999}]},{"osd":3,"last update":"Fri Sep 13 05:25:36 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46500000000000002,"15min":0.46100000000000002},"min":{"1min":0.36199999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.442,"5min":0.44700000000000001,"15min":0.44800000000000001},"min":{"1min":0.34000000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.41099999999999998}]},{"osd":4,"last update":"Fri Sep 13 05:25:02 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.624}]},{"osd":5,"last update":"Fri Sep 13 05:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.66800000000000004}]},{"osd":6,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.53300000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.38700000000000001,"15min":0.38},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.34699999999999998}]},{"osd":1,"last update":"Fri Sep 13 05:25:14 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.39000000000000001,"15min":0.378},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.315}]},{"osd":3,"last update":"Fri Sep 13 05:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44600000000000001,"15min":0.45000000000000001},"min":{"1min":0.315,"5min":0.312,"15min":0.312},"max":{"1min":0.54400000000000004,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.45300000000000001,"15min":0.45700000000000002},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61699999999999999,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.39800000000000002}]},{"osd":4,"last update":"Fri Sep 13 05:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.77500000000000002}]},{"osd":5,"last update":"Fri Sep 13 05:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.76300000000000001}]},{"osd":6,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.626},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.79800000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.60499999999999998,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.45400000000000001,"15min":0.46400000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.64800000000000002,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.312}]},{"osd":1,"last update":"Fri Sep 13 05:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.45500000000000002,"15min":0.45900000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58099999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.46999999999999997,"15min":0.47999999999999998},"min":{"1min":0.27900000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.627,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.34699999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.51400000000000001,"15min":0.52600000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58699999999999997,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.26600000000000001}]},{"osd":4,"last update":"Fri Sep 13 05:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.44600000000000001}]},{"osd":5,"last update":"Fri Sep 13 05:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.46000000000000002}]},{"osd":6,"last update":"Fri Sep 13 05:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57099999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.59999999999999998}]},{"osd":1,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.47799999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.63300000000000001}]},{"osd":3,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.68300000000000005}]},{"osd":5,"last update":"Fri Sep 13 05:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.66300000000000003}]},{"osd":6,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.50700000000000001}]},{"osd":1,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.495},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.56200000000000006}]},{"osd":2,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.58699999999999997}]},{"osd":3,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.53800000000000003}]},{"osd":4,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.371},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.432}]},{"osd":6,"last update":"Fri Sep 13 05:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.38500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T05:26:07.240 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T05:26:07.459 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:26:07.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:07 smithi053 ceph-mon[23721]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:07.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:07 smithi053 ceph-mon[23721]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:07 smithi110 ceph-mon[25804]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:07 smithi110 ceph-mon[25804]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:09.192 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:26:09.195 INFO:teuthology.orchestra.run.smithi053.stderr:dumped all 2024-09-13T05:26:09.835 INFO:teuthology.orchestra.run.smithi053.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-13T05:26:08.359905+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48880,"kb_used_data":6320,"kb_used_omap":0,"kb_used_meta":42496,"kb_avail":749912336,"statfs":{"total":767960285184,"available":767910232064,"internally_reserved":0,"allocated":6471680,"data_stored":3548002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43515904},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002151"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-13T05:25:12.548458+0000","last_change":"2024-09-13T05:25:12.548458+0000","last_active":"2024-09-13T05:25:12.548458+0000","last_peered":"2024-09-13T05:25:12.548458+0000","last_clean":"2024-09-13T05:25:12.548458+0000","last_became_active":"2024-09-13T05:25:12.547966+0000","last_became_peered":"2024-09-13T05:25:12.547966+0000","last_unstale":"2024-09-13T05:25:12.548458+0000","last_undegraded":"2024-09-13T05:25:12.548458+0000","last_fullsized":"2024-09-13T05:25:12.548458+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T05:23:08.807859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T05:23:08.807859+0000","last_clean_scrub_stamp":"2024-09-13T05:23:08.807859+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T15:10:52.923340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54500000000000004}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65100000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38700000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.438}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.50800000000000001}]},{"osd":1,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.056,"5min":1.056,"15min":1.056},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.60799999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58899999999999997}]},{"osd":3,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.68400000000000005}]},{"osd":4,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.74399999999999999}]},{"osd":5,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.72099999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81000000000000005}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6364,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738788,"statfs":{"total":95995035648,"available":95988518912,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.36599999999999999,"15min":0.35199999999999998},"min":{"1min":0.249,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"last":0.438},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.36499999999999999,"15min":0.34999999999999998},"min":{"1min":0.27000000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.34899999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:14 2024","interfaces":[{"interface":"back","average":{"1min":0.45000000000000001,"5min":0.39800000000000002,"15min":0.38900000000000001},"min":{"1min":0.28399999999999997,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38,"15min":0.372},"min":{"1min":0.32400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.42399999999999999}]},{"osd":3,"last update":"Fri Sep 13 05:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.441,"15min":0.436},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.45600000000000002}]},{"osd":4,"last update":"Fri Sep 13 05:25:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.61299999999999999}]},{"osd":5,"last update":"Fri Sep 13 05:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.54000000000000004}]},{"osd":6,"last update":"Fri Sep 13 05:25:46 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.59299999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 05:25:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.34000000000000002,"15min":0.32000000000000001},"min":{"1min":0.23300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.34699999999999998,"15min":0.315},"min":{"1min":0.248,"5min":0.22,"15min":0.22},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.374}]},{"osd":2,"last update":"Fri Sep 13 05:25:11 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.35699999999999998,"15min":0.34899999999999998},"min":{"1min":0.27700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.39700000000000002,"15min":0.39000000000000001},"min":{"1min":0.30499999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.43099999999999999}]},{"osd":3,"last update":"Fri Sep 13 05:25:36 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.46500000000000002,"15min":0.46100000000000002},"min":{"1min":0.36199999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.44700000000000001,"15min":0.44800000000000001},"min":{"1min":0.34000000000000002,"5min":0.254,"15min":0.254},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.33800000000000002}]},{"osd":4,"last update":"Fri Sep 13 05:26:03 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.628,"15min":0.629},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.629,"5min":0.60999999999999999,"15min":0.60699999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.66600000000000004}]},{"osd":5,"last update":"Fri Sep 13 05:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.75700000000000001}]},{"osd":6,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.77300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70699999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:14 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.38700000000000001,"15min":0.38},"min":{"1min":0.27500000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.34699999999999998}]},{"osd":1,"last update":"Fri Sep 13 05:25:14 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.39000000000000001,"15min":0.378},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.315}]},{"osd":3,"last update":"Fri Sep 13 05:25:39 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.44600000000000001,"15min":0.45000000000000001},"min":{"1min":0.315,"5min":0.312,"15min":0.312},"max":{"1min":0.54400000000000004,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.45300000000000001,"15min":0.45700000000000002},"min":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61699999999999999,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.39800000000000002}]},{"osd":4,"last update":"Fri Sep 13 05:24:58 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.77500000000000002}]},{"osd":5,"last update":"Fri Sep 13 05:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.76300000000000001}]},{"osd":6,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.626},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.79800000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60799999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149730,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.60499999999999998,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.45400000000000001,"15min":0.46400000000000002},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.64800000000000002,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.32400000000000001}]},{"osd":1,"last update":"Fri Sep 13 05:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.45500000000000002,"15min":0.45900000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58099999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.376},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.46999999999999997,"15min":0.47999999999999998},"min":{"1min":0.27900000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.627,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.34699999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:37 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.51400000000000001,"15min":0.52600000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.58699999999999997,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.41699999999999998}]},{"osd":4,"last update":"Fri Sep 13 05:26:04 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.59699999999999998,"15min":0.59499999999999997},"min":{"1min":0.39200000000000002,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.63900000000000001,"15min":0.64900000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.872,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.72099999999999997}]},{"osd":5,"last update":"Fri Sep 13 05:25:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.73399999999999999}]},{"osd":6,"last update":"Fri Sep 13 05:25:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.749}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.627}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.59999999999999998}]},{"osd":1,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.47799999999999998}]},{"osd":2,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.63300000000000001}]},{"osd":3,"last update":"Fri Sep 13 05:25:03 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.68300000000000005}]},{"osd":5,"last update":"Fri Sep 13 05:25:24 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.66300000000000003}]},{"osd":6,"last update":"Fri Sep 13 05:25:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.58799999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48199999999999998}]},{"osd":1,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.52600000000000002}]},{"osd":2,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.61299999999999999}]},{"osd":3,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.58499999999999996}]},{"osd":4,"last update":"Fri Sep 13 05:25:26 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.434},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.55700000000000005}]},{"osd":6,"last update":"Fri Sep 13 05:25:44 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.66800000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T05:26:09.837 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T05:26:09.837 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T05:26:09.837 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T05:26:09.838 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T05:26:10.068 INFO:teuthology.orchestra.run.smithi053.stderr:Inferring config /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/mon.smithi053/config 2024-09-13T05:26:10.191 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:09 smithi053 ceph-mon[23721]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:10.217 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:09 smithi110 ceph-mon[25804]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:11.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:10 smithi053 ceph-mon[23721]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:11.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:11.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:11.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:11.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:10 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:10 smithi110 ceph-mon[25804]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:11.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:10 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:11.832 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:26:11.833 INFO:teuthology.orchestra.run.smithi053.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T05:26:12.150 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:11 smithi053 ceph-mon[23721]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:11 smithi110 ceph-mon[25804]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:12.393 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T05:26:12.393 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T05:26:12.393 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T05:26:12.403 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:26:12.403 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-13T05:26:12.995 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:12 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3009366756' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T05:26:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:12 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3009366756' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T05:26:13.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:13 smithi053 ceph-mon[23721]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:14.181 INFO:teuthology.orchestra.run.smithi053.stdout:Backend: cephadm 2024-09-13T05:26:14.181 INFO:teuthology.orchestra.run.smithi053.stdout:Available: Yes 2024-09-13T05:26:14.181 INFO:teuthology.orchestra.run.smithi053.stdout:Paused: No 2024-09-13T05:26:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:13 smithi110 ceph-mon[25804]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:14.824 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-13T05:26:16.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:15 smithi053 ceph-mon[23721]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:16.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:15 smithi053 ceph-mon[23721]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:15 smithi110 ceph-mon[25804]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:15 smithi110 ceph-mon[25804]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:16.697 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:26:16.697 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (4m) 2m ago 5m 15.5M - ba2b418f427c 2a3ae76369c5 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (5m) 2m ago 5m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (5m) 69s ago 5m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (4m) 2m ago 5m 40.0M - 8.3.5 dad864ee21e9 41e920907455 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (7m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (5m) 69s ago 5m 411M - 17.2.0 e1d6a67b021e 69db5250e9d9 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (7m) 2m ago 7m 41.2M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (4m) 69s ago 4m 40.0M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (5m) 2m ago 5m 15.0M - 1dbe0e931976 aa1768a447f0 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (4m) 69s ago 4m 18.4M - 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (4m) 2m ago 4m 32.9M 3217M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (3m) 2m ago 3m 37.3M 3217M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (3m) 2m ago 3m 32.5M 3217M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:26:16.698 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (2m) 2m ago 2m 27.6M 3217M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:26:16.699 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (2m) 69s ago 2m 33.0M 3984M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:26:16.699 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (2m) 69s ago 2m 35.8M 3984M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:26:16.699 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (98s) 69s ago 98s 30.8M 3984M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:26:16.699 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (74s) 69s ago 74s 12.6M 3984M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:26:16.699 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (4m) 2m ago 4m 55.5M - 514e6a882f6e 7c997d72a66c 2024-09-13T05:26:17.320 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-13T05:26:17.964 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:17 smithi053 ceph-mon[23721]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:17.964 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:17 smithi053 ceph-mon[23721]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:18.045 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:17 smithi110 ceph-mon[25804]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:18.046 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:17 smithi110 ceph-mon[25804]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T05:26:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-13T05:26:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:crash 2/2 2m ago 6m * 2024-09-13T05:26:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-13T05:26:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:mgr 2/2 2m ago 6m count:2 2024-09-13T05:26:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:mon 2/2 2m ago 5m smithi053:172.21.15.53=smithi053;smithi110:172.21.15.110=smithi110;count:2 2024-09-13T05:26:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-13T05:26:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd 8 2m ago - 2024-09-13T05:26:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-13T05:26:19.825 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-13T05:26:19.924 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:19 smithi053 ceph-mon[23721]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:20.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:19 smithi110 ceph-mon[25804]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:21.066 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:20 smithi053 ceph-mon[23721]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:21.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:20 smithi110 ceph-mon[25804]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:21.586 INFO:teuthology.orchestra.run.smithi053.stdout:HOST ADDR LABELS STATUS 2024-09-13T05:26:21.586 INFO:teuthology.orchestra.run.smithi053.stdout:smithi053 172.21.15.53 2024-09-13T05:26:21.586 INFO:teuthology.orchestra.run.smithi053.stdout:smithi110 172.21.15.110 2024-09-13T05:26:21.586 INFO:teuthology.orchestra.run.smithi053.stdout:2 hosts in cluster 2024-09-13T05:26:22.147 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:21 smithi053 ceph-mon[23721]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:22.174 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-13T05:26:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:21 smithi110 ceph-mon[25804]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:23.071 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:22 smithi053 ceph-mon[23721]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:23.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:22 smithi110 ceph-mon[25804]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:23.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:23 smithi053 ceph-mon[23721]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:24.027 INFO:teuthology.orchestra.run.smithi053.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-13T05:26:24.027 INFO:teuthology.orchestra.run.smithi053.stdout:smithi053 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB751300G8480DGN 480G LVM detected, locked 2024-09-13T05:26:24.027 INFO:teuthology.orchestra.run.smithi053.stdout:smithi110 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB7476002T480DGN 480G LVM detected, locked 2024-09-13T05:26:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:23 smithi110 ceph-mon[25804]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:24.703 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T05:26:24.722 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:26:24.723 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T05:26:24.750 INFO:teuthology.orchestra.run.smithi053.stderr:+ systemctl stop nfs-server 2024-09-13T05:26:24.758 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi110.front.sepia.ceph.com 2024-09-13T05:26:24.759 DEBUG:teuthology.orchestra.run.smithi110:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T05:26:24.790 INFO:teuthology.orchestra.run.smithi110.stderr:+ systemctl stop nfs-server 2024-09-13T05:26:24.798 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T05:26:24.807 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:26:24.807 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-13T05:26:25.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:25 smithi053 ceph-mon[23721]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:25.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:25 smithi053 ceph-mon[23721]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:26.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:25 smithi110 ceph-mon[25804]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:26.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:25 smithi110 ceph-mon[25804]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:27 smithi110 ceph-mon[25804]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:27 smithi110 ceph-mon[25804]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:27 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T05:26:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:27 smithi053 ceph-mon[23721]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:27 smithi053 ceph-mon[23721]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:27 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T05:26:29.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:28 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T05:26:29.151 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:28 smithi053 ceph-mon[23721]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T05:26:29.152 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:28 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T05:26:29.152 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:28 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[23717]: 2024-09-13T05:26:28.809+0000 7fdea541d700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:26:29.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:28 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T05:26:29.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:28 smithi110 ceph-mon[25804]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T05:26:29.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:28 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T05:26:29.540 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-13T05:26:29.549 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-13T05:26:29.550 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T05:26:30.066 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: pgmap v213: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: fsmap foofs:0 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: Saving service mds.foofs spec with placement count:2 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:30.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:30.068 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:30.068 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:30.068 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:30.068 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T05:26:30.068 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T05:26:30.068 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:30.068 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:29 smithi053 ceph-mon[23721]: Deploying daemon mds.foofs.smithi053.rzvrng on smithi053 2024-09-13T05:26:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: pgmap v213: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T05:26:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: fsmap foofs:0 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: Saving service mds.foofs spec with placement count:2 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:30.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T05:26:30.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T05:26:30.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:30.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:29 smithi110 ceph-mon[25804]: Deploying daemon mds.foofs.smithi053.rzvrng on smithi053 2024-09-13T05:26:30.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:30 smithi053 ceph-mon[23721]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T05:26:31.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:30 smithi110 ceph-mon[25804]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T05:26:32.083 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:31 smithi053 ceph-mon[23721]: pgmap v217: 65 pgs: 25 active+clean, 3 creating+activating, 10 creating+peering, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:32.083 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:31 smithi053 ceph-mon[23721]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T05:26:32.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:31 smithi110 ceph-mon[25804]: pgmap v217: 65 pgs: 25 active+clean, 3 creating+activating, 10 creating+peering, 27 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:32.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:31 smithi110 ceph-mon[25804]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T05:26:32.547 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:26:32.548 INFO:teuthology.orchestra.run.smithi053.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T05:19:56.091160Z", "last_refresh": "2024-09-13T05:23:44.848545Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T05:19:49.578973Z", "last_refresh": "2024-09-13T05:23:44.848610Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T05:19:52.700899Z", "last_refresh": "2024-09-13T05:23:44.848672Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T05:26:28.830509Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T05:26:28.825146Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T05:19:47.487653Z", "last_refresh": "2024-09-13T05:23:44.848474Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi053:172.21.15.53=smithi053", "smithi110:172.21.15.110=smithi110"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T05:20:40.946189Z", "last_refresh": "2024-09-13T05:23:44.848327Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T05:19:54.826039Z", "last_refresh": "2024-09-13T05:23:44.848745Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T05:23:44.848876Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T05:19:51.449591Z", "last_refresh": "2024-09-13T05:23:44.848814Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T05:26:33.374 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: pgmap v219: 65 pgs: 34 active+clean, 3 creating+activating, 10 creating+peering, 18 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: Deploying daemon mds.foofs.smithi110.akbygi on smithi110 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: mds.? [v2:172.21.15.53:6834/2495430984,v1:172.21.15.53:6835/2495430984] up:boot 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: daemon mds.foofs.smithi053.rzvrng assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T05:26:34.191 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: Cluster is now healthy 2024-09-13T05:26:34.192 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: fsmap foofs:0 1 up:standby 2024-09-13T05:26:34.192 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"}]: dispatch 2024-09-13T05:26:34.192 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:creating} 2024-09-13T05:26:34.192 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:33 smithi110 ceph-mon[25804]: daemon mds.foofs.smithi053.rzvrng is now active in filesystem foofs as rank 0 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: pgmap v219: 65 pgs: 34 active+clean, 3 creating+activating, 10 creating+peering, 18 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: from='client.14502 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: Deploying daemon mds.foofs.smithi110.akbygi on smithi110 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: mds.? [v2:172.21.15.53:6834/2495430984,v1:172.21.15.53:6835/2495430984] up:boot 2024-09-13T05:26:34.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: daemon mds.foofs.smithi053.rzvrng assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T05:26:34.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T05:26:34.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T05:26:34.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: Cluster is now healthy 2024-09-13T05:26:34.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: fsmap foofs:0 1 up:standby 2024-09-13T05:26:34.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"}]: dispatch 2024-09-13T05:26:34.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:creating} 2024-09-13T05:26:34.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:33 smithi053 ceph-mon[23721]: daemon mds.foofs.smithi053.rzvrng is now active in filesystem foofs as rank 0 2024-09-13T05:26:34.375 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T05:26:34.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:34 smithi053 ceph-mon[23721]: mds.? [v2:172.21.15.53:6834/2495430984,v1:172.21.15.53:6835/2495430984] up:active 2024-09-13T05:26:34.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:34 smithi053 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 2024-09-13T05:26:35.108 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:34 smithi110 ceph-mon[25804]: mds.? [v2:172.21.15.53:6834/2495430984,v1:172.21.15.53:6835/2495430984] up:active 2024-09-13T05:26:35.108 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:34 smithi110 ceph-mon[25804]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: pgmap v220: 65 pgs: 56 active+clean, 3 creating+activating, 6 creating+peering; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 184 B/s wr, 2 op/s 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:boot 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:26:36.419 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"}]: dispatch 2024-09-13T05:26:36.420 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:36 smithi110 ceph-mon[25804]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:26:36.449 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:26:36.449 INFO:teuthology.orchestra.run.smithi053.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T05:19:56.091160Z", "last_refresh": "2024-09-13T05:23:44.848545Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T05:19:49.578973Z", "last_refresh": "2024-09-13T05:23:44.848610Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T05:19:52.700899Z", "last_refresh": "2024-09-13T05:23:44.848672Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T05:26:28.830509Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T05:26:28.825146Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T05:19:47.487653Z", "last_refresh": "2024-09-13T05:23:44.848474Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi053:172.21.15.53=smithi053", "smithi110:172.21.15.110=smithi110"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T05:20:40.946189Z", "last_refresh": "2024-09-13T05:23:44.848327Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T05:19:54.826039Z", "last_refresh": "2024-09-13T05:23:44.848745Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T05:23:44.848876Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T05:19:51.449591Z", "last_refresh": "2024-09-13T05:23:44.848814Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T05:26:36.449 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: pgmap v220: 65 pgs: 56 active+clean, 3 creating+activating, 6 creating+peering; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 184 B/s wr, 2 op/s 2024-09-13T05:26:36.449 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:36.449 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:36.449 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:36.449 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:36.450 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:boot 2024-09-13T05:26:36.450 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:26:36.450 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"}]: dispatch 2024-09-13T05:26:36.450 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:36 smithi053 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:26:37.612 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T05:26:38.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:38 smithi053 ceph-mon[23721]: pgmap v221: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 813 B/s wr, 4 op/s 2024-09-13T05:26:38.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:38 smithi053 ceph-mon[23721]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:38.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:38 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:38.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:38 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:38.396 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:38 smithi110 ceph-mon[25804]: pgmap v221: 65 pgs: 65 active+clean; 578 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 813 B/s wr, 4 op/s 2024-09-13T05:26:38.396 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:38 smithi110 ceph-mon[25804]: from='client.14514 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:38.396 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:38 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:38.396 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:38 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:38.613 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T05:26:40.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:40 smithi053 ceph-mon[23721]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-09-13T05:26:40.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:40 smithi110 ceph-mon[25804]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-09-13T05:26:41.591 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:26:41.592 INFO:teuthology.orchestra.run.smithi053.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T05:19:56.091160Z", "last_refresh": "2024-09-13T05:23:44.848545Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T05:19:49.578973Z", "last_refresh": "2024-09-13T05:23:44.848610Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T05:19:52.700899Z", "last_refresh": "2024-09-13T05:23:44.848672Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T05:26:28.830509Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T05:26:28.825146Z", "last_refresh": "2024-09-13T05:26:37.134787Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T05:19:47.487653Z", "last_refresh": "2024-09-13T05:23:44.848474Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi053:172.21.15.53=smithi053", "smithi110:172.21.15.110=smithi110"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T05:20:40.946189Z", "last_refresh": "2024-09-13T05:23:44.848327Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T05:19:54.826039Z", "last_refresh": "2024-09-13T05:23:44.848745Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T05:23:44.848876Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T05:19:51.449591Z", "last_refresh": "2024-09-13T05:23:44.848814Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T05:26:42.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:42 smithi053 ceph-mon[23721]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T05:26:42.392 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-13T05:26:42.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:42 smithi110 ceph-mon[25804]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-13T05:26:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:43.393 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T05:26:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='client.14518 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:43.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:44.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:44 smithi110 ceph-mon[25804]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T05:26:44.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:44 smithi053 ceph-mon[23721]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T05:26:45.369 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:26:45.369 INFO:teuthology.orchestra.run.smithi053.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T05:19:56.091160Z", "last_refresh": "2024-09-13T05:26:42.940174Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T05:19:49.578973Z", "last_refresh": "2024-09-13T05:26:37.133642Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T05:19:52.700899Z", "last_refresh": "2024-09-13T05:26:42.940306Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T05:26:28.830509Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T05:26:28.825146Z", "last_refresh": "2024-09-13T05:26:37.134787Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T05:19:47.487653Z", "last_refresh": "2024-09-13T05:26:37.133888Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi053:172.21.15.53=smithi053", "smithi110:172.21.15.110=smithi110"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T05:20:40.946189Z", "last_refresh": "2024-09-13T05:26:37.134031Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T05:19:54.826039Z", "last_refresh": "2024-09-13T05:26:37.134182Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T05:26:37.134307Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T05:19:51.449591Z", "last_refresh": "2024-09-13T05:26:42.940431Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T05:26:45.730 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:45 smithi053 ceph-mon[23721]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T05:26:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:45 smithi110 ceph-mon[25804]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T05:26:46.021 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-13T05:26:46.021 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T05:26:46.030 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:26:46.031 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-13T05:26:46.839 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:46 smithi053 ceph-mon[23721]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:46.871 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:46 smithi110 ceph-mon[25804]: from='client.14522 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:26:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:47 smithi110 ceph-mon[25804]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 2 op/s 2024-09-13T05:26:47.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:47 smithi053 ceph-mon[23721]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 2 op/s 2024-09-13T05:26:48.646 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:48 smithi110 ceph-mon[25804]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:48.646 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:48 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T05:26:48.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:48 smithi053 ceph-mon[23721]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:48.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:48 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T05:26:49.613 INFO:teuthology.orchestra.run.smithi053.stdout:NFS Cluster Created Successfully 2024-09-13T05:26:49.830 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:49 smithi053 ceph-mon[23721]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 597 B/s wr, 0 op/s 2024-09-13T05:26:49.830 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T05:26:49.830 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:49 smithi053 ceph-mon[23721]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T05:26:49.830 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T05:26:49.832 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:49 smithi110 ceph-mon[25804]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 597 B/s wr, 0 op/s 2024-09-13T05:26:49.833 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T05:26:49.833 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:49 smithi110 ceph-mon[25804]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T05:26:49.833 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T05:26:50.388 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-13T05:26:50.631 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T05:26:50.631 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: Saving service nfs.foo spec with placement count:2 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T05:26:50.632 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: Rados config object exists: conf-nfs.foo 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx-rgw 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:50.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:50 smithi053 ceph-mon[23721]: Deploying daemon nfs.foo.0.0.smithi053.zidqnx on smithi053 2024-09-13T05:26:50.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T05:26:50.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T05:26:50.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: Saving service nfs.foo spec with placement count:2 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:50.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T05:26:50.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:26:50.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: Rados config object exists: conf-nfs.foo 2024-09-13T05:26:50.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx-rgw 2024-09-13T05:26:50.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T05:26:50.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T05:26:50.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:50.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:50 smithi110 ceph-mon[25804]: Deploying daemon nfs.foo.0.0.smithi053.zidqnx on smithi053 2024-09-13T05:26:51.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:51 smithi053 ceph-mon[23721]: pgmap v230: 97 pgs: 3 creating+activating, 10 creating+peering, 14 unknown, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:51.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:51 smithi053 ceph-mon[23721]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T05:26:51.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:51 smithi053 ceph-mon[23721]: mgrmap e20: smithi053.ygzjut(active, since 5m), standbys: smithi110.rivldx 2024-09-13T05:26:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:51 smithi110 ceph-mon[25804]: pgmap v230: 97 pgs: 3 creating+activating, 10 creating+peering, 14 unknown, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T05:26:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:51 smithi110 ceph-mon[25804]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T05:26:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:51 smithi110 ceph-mon[25804]: mgrmap e20: smithi053.ygzjut(active, since 5m), standbys: smithi110.rivldx 2024-09-13T05:26:53.486 INFO:teuthology.orchestra.run.smithi053.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-13T05:26:53.486 INFO:teuthology.orchestra.run.smithi053.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-13T05:26:53.486 INFO:teuthology.orchestra.run.smithi053.stderr:Error EINVAL: invalid command 2024-09-13T05:26:53.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:53 smithi053 ceph-mon[23721]: pgmap v232: 97 pgs: 3 creating+activating, 10 creating+peering, 7 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:26:53.849 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:26:53.849 INFO:teuthology.orchestra.run.smithi053.stdout: "bind": "/fake", 2024-09-13T05:26:53.849 INFO:teuthology.orchestra.run.smithi053.stdout: "fs": "foofs", 2024-09-13T05:26:53.850 INFO:teuthology.orchestra.run.smithi053.stdout: "path": "/", 2024-09-13T05:26:53.850 INFO:teuthology.orchestra.run.smithi053.stdout: "cluster": "foo", 2024-09-13T05:26:53.850 INFO:teuthology.orchestra.run.smithi053.stdout: "mode": "RW" 2024-09-13T05:26:53.850 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:26:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:53 smithi110 ceph-mon[25804]: pgmap v232: 97 pgs: 3 creating+activating, 10 creating+peering, 7 unknown, 77 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:26:54.526 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T05:26:55.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:54 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T05:26:55.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T05:26:55.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:54 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T05:26:56.153 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:55 smithi053 ceph-mon[23721]: pgmap v233: 97 pgs: 3 creating+activating, 94 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:26:56.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:55 smithi110 ceph-mon[25804]: pgmap v233: 97 pgs: 3 creating+activating, 94 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:26:56.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T05:26:56.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:26:56.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T05:26:56.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T05:26:56.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:56 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:57.119 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T05:26:57.119 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:26:57.119 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T05:26:57.119 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T05:26:57.119 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:56 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:58.236 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:57 smithi110 ceph-mon[25804]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 131 B/s rd, 394 B/s wr, 0 op/s 2024-09-13T05:26:58.236 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:57 smithi110 ceph-mon[25804]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:58.236 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:57 smithi110 ceph-mon[25804]: Rados config object exists: conf-nfs.foo 2024-09-13T05:26:58.236 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:57 smithi110 ceph-mon[25804]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo-rgw 2024-09-13T05:26:58.236 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:57 smithi110 ceph-mon[25804]: Deploying daemon nfs.foo.1.0.smithi110.fvztpo on smithi110 2024-09-13T05:26:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:57 smithi053 ceph-mon[23721]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 131 B/s rd, 394 B/s wr, 0 op/s 2024-09-13T05:26:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:57 smithi053 ceph-mon[23721]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:57 smithi053 ceph-mon[23721]: Rados config object exists: conf-nfs.foo 2024-09-13T05:26:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:57 smithi053 ceph-mon[23721]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo-rgw 2024-09-13T05:26:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:57 smithi053 ceph-mon[23721]: Deploying daemon nfs.foo.1.0.smithi110.fvztpo on smithi110 2024-09-13T05:26:58.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:58 smithi110 ceph-mon[25804]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:58.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-09-13T05:26:58.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-13T05:26:58.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T05:26:58.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-09-13T05:26:58.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:26:58.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:26:59.119 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:58 smithi053 ceph-mon[23721]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:26:59.119 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]: dispatch 2024-09-13T05:26:59.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-13T05:26:59.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]: dispatch 2024-09-13T05:26:59.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-09-13T05:26:59.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:26:59.120 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:26:59.874 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]': finished 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:59.875 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:26:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:26:59.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.9", "id": [7, 2]}]': finished 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 2]}]': finished 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:26:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:26:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:27:01.155 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:00 smithi053 ceph-mon[23721]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:01.155 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:00 smithi053 ceph-mon[23721]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T05:27:01.213 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:00 smithi110 ceph-mon[25804]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:01.213 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:00 smithi110 ceph-mon[25804]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T05:27:02.118 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:01 smithi053 ceph-mon[23721]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.1 KiB/s wr, 4 op/s 2024-09-13T05:27:02.118 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:01 smithi053 ceph-mon[23721]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:02.118 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:01 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:01 smithi110 ceph-mon[25804]: pgmap v238: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.1 KiB/s wr, 4 op/s 2024-09-13T05:27:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:01 smithi110 ceph-mon[25804]: from='client.14594 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:01 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:03.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:02 smithi110 ceph-mon[25804]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:03.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:03.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:02 smithi053 ceph-mon[23721]: from='client.14598 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:03.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:04.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:03 smithi110 ceph-mon[25804]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-13T05:27:04.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:03 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:04.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:03 smithi053 ceph-mon[23721]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-13T05:27:04.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:03 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:04.532 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo ?:2049 2/2 2s ago 14s count:2 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:04 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.214 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T05:27:05.228 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:27:05.229 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-13T05:27:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:27:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:27:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:27:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:04 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:05.283 INFO:teuthology.orchestra.run.smithi053.stderr:+ mkdir /mnt/foo 2024-09-13T05:27:05.291 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-13T05:27:05.357 INFO:teuthology.orchestra.run.smithi053.stderr:++ hostname 2024-09-13T05:27:05.358 INFO:teuthology.orchestra.run.smithi053.stderr:+ mount -t nfs smithi053:/fake /mnt/foo -o sync 2024-09-13T05:27:05.809 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-13T05:27:05.875 INFO:teuthology.orchestra.run.smithi053.stderr:+ echo test 2024-09-13T05:27:05.897 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-13T05:27:05.959 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:05 smithi053 ceph-mon[23721]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.0 KiB/s wr, 8 op/s 2024-09-13T05:27:05.960 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:05 smithi053 ceph-mon[23721]: from='client.14606 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:05.962 INFO:teuthology.orchestra.run.smithi053.stderr:+ sync 2024-09-13T05:27:06.196 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T05:27:06.207 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T05:27:06.207 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T05:27:06.208 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T05:27:06.208 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:27:06.208 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T05:27:06.209 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T05:27:06.209 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-13T05:27:06.210 INFO:teuthology.task.exec:Executing custom commands... 2024-09-13T05:27:06.210 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:27:06.210 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-13T05:27:06.243 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:05 smithi110 ceph-mon[25804]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.0 KiB/s wr, 8 op/s 2024-09-13T05:27:06.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:05 smithi110 ceph-mon[25804]: from='client.14606 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:06.275 INFO:teuthology.orchestra.run.smithi053.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T05:27:06.275 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:27:06.276 INFO:teuthology.orchestra.run.smithi053.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-13T05:27:06.277 INFO:teuthology.orchestra.run.smithi053.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-13T05:27:06.377 INFO:teuthology.orchestra.run.smithi053.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T05:27:06.377 INFO:teuthology.orchestra.run.smithi053.stdout:releasing clients 2024-09-13T05:27:07.377 INFO:teuthology.orchestra.run.smithi053.stdout: 5 161 33.30 MB/sec warmup 1 sec latency 23.630 ms 2024-09-13T05:27:08.114 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:07 smithi053 ceph-mon[23721]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-13T05:27:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:07 smithi110 ceph-mon[25804]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-09-13T05:27:08.377 INFO:teuthology.orchestra.run.smithi053.stdout: 5 296 27.28 MB/sec warmup 2 sec latency 31.240 ms 2024-09-13T05:27:09.377 INFO:teuthology.orchestra.run.smithi053.stdout: 5 519 29.93 MB/sec warmup 3 sec latency 23.653 ms 2024-09-13T05:27:09.998 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T05:27:10.161 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:09 smithi053 ceph-mon[23721]: pgmap v242: 97 pgs: 97 active+clean; 15 MiB data, 63 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-13T05:27:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:09 smithi110 ceph-mon[25804]: pgmap v242: 97 pgs: 97 active+clean; 15 MiB data, 63 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.5 MiB/s wr, 31 op/s 2024-09-13T05:27:10.377 INFO:teuthology.orchestra.run.smithi053.stdout: 5 743 26.63 MB/sec warmup 4 sec latency 30.172 ms 2024-09-13T05:27:11.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 1108 22.96 MB/sec warmup 5 sec latency 47.000 ms 2024-09-13T05:27:11.944 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:11 smithi053 ceph-mon[23721]: pgmap v243: 97 pgs: 97 active+clean; 60 MiB data, 97 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.6 MiB/s wr, 247 op/s 2024-09-13T05:27:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:11 smithi110 ceph-mon[25804]: pgmap v243: 97 pgs: 97 active+clean; 60 MiB data, 97 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 5.6 MiB/s wr, 247 op/s 2024-09-13T05:27:12.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 1965 22.08 MB/sec warmup 6 sec latency 58.604 ms 2024-09-13T05:27:13.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 2706 21.66 MB/sec warmup 7 sec latency 32.397 ms 2024-09-13T05:27:13.988 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:13 smithi053 ceph-mon[23721]: pgmap v244: 97 pgs: 97 active+clean; 79 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 6.8 MiB/s wr, 369 op/s 2024-09-13T05:27:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:13 smithi110 ceph-mon[25804]: pgmap v244: 97 pgs: 97 active+clean; 79 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 6.8 MiB/s wr, 369 op/s 2024-09-13T05:27:14.352 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T05:27:14.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 3356 22.48 MB/sec warmup 8 sec latency 32.589 ms 2024-09-13T05:27:15.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 3778 21.99 MB/sec warmup 9 sec latency 38.306 ms 2024-09-13T05:27:16.047 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:15 smithi053 ceph-mon[23721]: pgmap v245: 97 pgs: 97 active+clean; 111 MiB data, 183 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 9.7 MiB/s wr, 451 op/s 2024-09-13T05:27:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:15 smithi110 ceph-mon[25804]: pgmap v245: 97 pgs: 97 active+clean; 111 MiB data, 183 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 9.7 MiB/s wr, 451 op/s 2024-09-13T05:27:16.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 4044 20.04 MB/sec warmup 10 sec latency 41.225 ms 2024-09-13T05:27:17.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 4663 19.07 MB/sec warmup 11 sec latency 32.480 ms 2024-09-13T05:27:18.058 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:17 smithi053 ceph-mon[23721]: pgmap v246: 97 pgs: 97 active+clean; 120 MiB data, 208 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 11 MiB/s wr, 596 op/s 2024-09-13T05:27:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:17 smithi110 ceph-mon[25804]: pgmap v246: 97 pgs: 97 active+clean; 120 MiB data, 208 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 11 MiB/s wr, 596 op/s 2024-09-13T05:27:18.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 5567 19.02 MB/sec warmup 12 sec latency 24.363 ms 2024-09-13T05:27:19.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 6033 18.25 MB/sec warmup 13 sec latency 53.481 ms 2024-09-13T05:27:19.960 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-13T05:27:20.111 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:19 smithi053 ceph-mon[23721]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 217 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 13 MiB/s wr, 726 op/s 2024-09-13T05:27:20.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:19 smithi110 ceph-mon[25804]: pgmap v247: 97 pgs: 97 active+clean; 130 MiB data, 217 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 13 MiB/s wr, 726 op/s 2024-09-13T05:27:20.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 6700 19.03 MB/sec warmup 14 sec latency 52.934 ms 2024-09-13T05:27:21.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 7267 19.38 MB/sec warmup 15 sec latency 71.345 ms 2024-09-13T05:27:22.160 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:21 smithi053 ceph-mon[23721]: pgmap v248: 97 pgs: 97 active+clean; 130 MiB data, 279 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 13 MiB/s wr, 848 op/s 2024-09-13T05:27:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:21 smithi110 ceph-mon[25804]: pgmap v248: 97 pgs: 97 active+clean; 130 MiB data, 279 MiB used, 715 GiB / 715 GiB avail; 5.9 KiB/s rd, 13 MiB/s wr, 848 op/s 2024-09-13T05:27:22.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 7418 18.43 MB/sec warmup 16 sec latency 71.748 ms 2024-09-13T05:27:23.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 7623 17.42 MB/sec warmup 17 sec latency 122.889 ms 2024-09-13T05:27:23.957 INFO:teuthology.orchestra.run.smithi053.stderr:module 'nfs' is already enabled 2024-09-13T05:27:24.015 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:23 smithi053 ceph-mon[23721]: pgmap v249: 97 pgs: 97 active+clean; 133 MiB data, 298 MiB used, 715 GiB / 715 GiB avail; 6.1 KiB/s rd, 10 MiB/s wr, 770 op/s 2024-09-13T05:27:24.016 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:23 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3565196415' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T05:27:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:23 smithi110 ceph-mon[25804]: pgmap v249: 97 pgs: 97 active+clean; 133 MiB data, 298 MiB used, 715 GiB / 715 GiB avail; 6.1 KiB/s rd, 10 MiB/s wr, 770 op/s 2024-09-13T05:27:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:23 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3565196415' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T05:27:24.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 7912 16.58 MB/sec warmup 18 sec latency 71.892 ms 2024-09-13T05:27:25.045 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:24 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3565196415' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T05:27:25.046 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:24 smithi053 ceph-mon[23721]: mgrmap e21: smithi053.ygzjut(active, since 6m), standbys: smithi110.rivldx 2024-09-13T05:27:25.088 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T05:27:25.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:24 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3565196415' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T05:27:25.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:24 smithi110 ceph-mon[25804]: mgrmap e21: smithi053.ygzjut(active, since 6m), standbys: smithi110.rivldx 2024-09-13T05:27:25.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 8192 16.06 MB/sec warmup 19 sec latency 52.335 ms 2024-09-13T05:27:26.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:25 smithi053 ceph-mon[23721]: pgmap v250: 97 pgs: 97 active+clean; 135 MiB data, 320 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.7 MiB/s wr, 668 op/s 2024-09-13T05:27:26.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:25 smithi110 ceph-mon[25804]: pgmap v250: 97 pgs: 97 active+clean; 135 MiB data, 320 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 9.7 MiB/s wr, 668 op/s 2024-09-13T05:27:26.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 8440 15.63 MB/sec warmup 20 sec latency 54.699 ms 2024-09-13T05:27:27.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 8796 15.35 MB/sec warmup 21 sec latency 62.269 ms 2024-09-13T05:27:27.986 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:27 smithi053 ceph-mon[23721]: pgmap v251: 97 pgs: 97 active+clean; 146 MiB data, 349 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.1 MiB/s wr, 739 op/s 2024-09-13T05:27:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:27 smithi110 ceph-mon[25804]: pgmap v251: 97 pgs: 97 active+clean; 146 MiB data, 349 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.1 MiB/s wr, 739 op/s 2024-09-13T05:27:28.248 INFO:teuthology.orchestra.run.smithi053.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 2024-09-13T05:27:28.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 9564 15.12 MB/sec warmup 22 sec latency 55.782 ms 2024-09-13T05:27:29.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 10066 15.50 MB/sec warmup 23 sec latency 32.996 ms 2024-09-13T05:27:29.450 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: from='client.24409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: pgmap v252: 97 pgs: 97 active+clean; 139 MiB data, 373 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.9 MiB/s wr, 626 op/s 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:29.451 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:29 smithi053 ceph-mon[23721]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 2024-09-13T05:27:29.481 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T05:27:29.481 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:27:29.482 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-13T05:27:29.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: from='client.24409 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:29.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 2024-09-13T05:27:29.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:29.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:27:29.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:27:29.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:27:29.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: pgmap v252: 97 pgs: 97 active+clean; 139 MiB data, 373 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.9 MiB/s wr, 626 op/s 2024-09-13T05:27:29.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:29.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:27:29.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:29 smithi110 ceph-mon[25804]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 2024-09-13T05:27:30.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 10522 15.48 MB/sec warmup 24 sec latency 32.385 ms 2024-09-13T05:27:31.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 10924 15.61 MB/sec warmup 25 sec latency 52.770 ms 2024-09-13T05:27:31.741 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:31 smithi053 ceph-mon[23721]: pgmap v253: 97 pgs: 97 active+clean; 142 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 667 op/s 2024-09-13T05:27:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:31 smithi110 ceph-mon[25804]: pgmap v253: 97 pgs: 97 active+clean; 142 MiB data, 413 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 667 op/s 2024-09-13T05:27:32.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 11145 15.07 MB/sec warmup 26 sec latency 44.655 ms 2024-09-13T05:27:33.080 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:27:33.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 11652 14.71 MB/sec warmup 27 sec latency 43.185 ms 2024-09-13T05:27:33.751 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:33 smithi053 ceph-mon[23721]: pgmap v254: 97 pgs: 97 active+clean; 143 MiB data, 416 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 540 op/s 2024-09-13T05:27:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:33 smithi110 ceph-mon[25804]: pgmap v254: 97 pgs: 97 active+clean; 143 MiB data, 416 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 540 op/s 2024-09-13T05:27:34.223 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:27:34.223 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (5m) 30s ago 7m 24.6M - ba2b418f427c 2a3ae76369c5 2024-09-13T05:27:34.223 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (6m) 30s ago 6m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:27:34.223 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (6m) 32s ago 6m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (5m) 30s ago 6m 49.6M - 8.3.5 dad864ee21e9 41e920907455 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (62s) 30s ago 61s 14.8M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (59s) 32s ago 59s 13.4M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (8m) 30s ago 8m 460M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (6m) 32s ago 6m 411M - 17.2.0 e1d6a67b021e 69db5250e9d9 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (8m) 30s ago 8m 43.6M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (6m) 32s ago 6m 37.6M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (40s) 30s ago 40s 49.1M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (35s) 32s ago 35s 46.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (6m) 30s ago 6m 15.0M - 1dbe0e931976 aa1768a447f0 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (6m) 32s ago 6m 18.6M - 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:27:34.224 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (5m) 30s ago 5m 42.3M 3217M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (5m) 30s ago 5m 46.0M 3217M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (4m) 30s ago 4m 40.3M 3217M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (4m) 30s ago 4m 42.8M 3217M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (3m) 32s ago 3m 40.2M 3984M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (3m) 32s ago 3m 41.9M 3984M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (2m) 32s ago 2m 39.4M 3984M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (2m) 32s ago 2m 41.1M 3984M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:27:34.225 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (5m) 30s ago 5m 62.1M - 514e6a882f6e 7c997d72a66c 2024-09-13T05:27:34.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 12201 14.90 MB/sec warmup 28 sec latency 26.807 ms 2024-09-13T05:27:34.755 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:34 smithi053 ceph-mon[23721]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:34.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:34 smithi053 ceph-mon[23721]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:27:34.916 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:27:34.917 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:27:34.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:34 smithi110 ceph-mon[25804]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:35.006 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:34 smithi110 ceph-mon[25804]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:35.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 12812 14.53 MB/sec warmup 29 sec latency 28.185 ms 2024-09-13T05:27:35.477 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:27:35.477 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3", 2024-09-13T05:27:35.477 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:27:35.477 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:27:35.477 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "", 2024-09-13T05:27:35.477 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 image" 2024-09-13T05:27:35.478 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:27:35.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:35 smithi053 ceph-mon[23721]: from='client.24421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:35.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:35 smithi053 ceph-mon[23721]: pgmap v255: 97 pgs: 97 active+clean; 155 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 584 op/s 2024-09-13T05:27:35.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:35 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1282489706' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:27:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:35 smithi110 ceph-mon[25804]: from='client.24421 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:35 smithi110 ceph-mon[25804]: pgmap v255: 97 pgs: 97 active+clean; 155 MiB data, 429 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 584 op/s 2024-09-13T05:27:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:35 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1282489706' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:27:36.107 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:27:36.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 13077 14.25 MB/sec warmup 30 sec latency 44.915 ms 2024-09-13T05:27:36.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:36 smithi053 ceph-mon[23721]: from='client.24429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:36.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:36 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/937479326' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:27:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:36 smithi110 ceph-mon[25804]: from='client.24429 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:27:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:36 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/937479326' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:27:37.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 13646 14.68 MB/sec warmup 31 sec latency 23.194 ms 2024-09-13T05:27:37.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:37 smithi053 ceph-mon[23721]: pgmap v256: 97 pgs: 97 active+clean; 156 MiB data, 453 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-09-13T05:27:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:37 smithi110 ceph-mon[25804]: pgmap v256: 97 pgs: 97 active+clean; 156 MiB data, 453 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 613 op/s 2024-09-13T05:27:38.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 14208 14.91 MB/sec warmup 32 sec latency 33.537 ms 2024-09-13T05:27:39.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 14537 14.73 MB/sec warmup 33 sec latency 63.957 ms 2024-09-13T05:27:39.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:39 smithi053 ceph-mon[23721]: pgmap v257: 97 pgs: 97 active+clean; 159 MiB data, 488 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 597 op/s 2024-09-13T05:27:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:39 smithi110 ceph-mon[25804]: pgmap v257: 97 pgs: 97 active+clean; 159 MiB data, 488 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 597 op/s 2024-09-13T05:27:40.378 INFO:teuthology.orchestra.run.smithi053.stdout: 5 15140 14.47 MB/sec warmup 34 sec latency 65.405 ms 2024-09-13T05:27:41.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 15839 14.67 MB/sec warmup 35 sec latency 34.119 ms 2024-09-13T05:27:41.839 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:41 smithi053 ceph-mon[23721]: pgmap v258: 97 pgs: 97 active+clean; 162 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 660 op/s 2024-09-13T05:27:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:41 smithi110 ceph-mon[25804]: pgmap v258: 97 pgs: 97 active+clean; 162 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 660 op/s 2024-09-13T05:27:42.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 16604 14.52 MB/sec warmup 36 sec latency 20.215 ms 2024-09-13T05:27:43.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 17188 14.86 MB/sec warmup 37 sec latency 33.148 ms 2024-09-13T05:27:43.854 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:43 smithi053 ceph-mon[23721]: pgmap v259: 97 pgs: 97 active+clean; 164 MiB data, 534 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 510 op/s 2024-09-13T05:27:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:43 smithi110 ceph-mon[25804]: pgmap v259: 97 pgs: 97 active+clean; 164 MiB data, 534 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.1 MiB/s wr, 510 op/s 2024-09-13T05:27:44.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 17898 15.21 MB/sec warmup 38 sec latency 33.564 ms 2024-09-13T05:27:45.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 18170 14.94 MB/sec warmup 39 sec latency 50.456 ms 2024-09-13T05:27:45.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:45 smithi053 ceph-mon[23721]: pgmap v260: 97 pgs: 97 active+clean; 167 MiB data, 560 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 669 op/s 2024-09-13T05:27:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:45 smithi110 ceph-mon[25804]: pgmap v260: 97 pgs: 97 active+clean; 167 MiB data, 560 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 669 op/s 2024-09-13T05:27:46.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 18490 14.61 MB/sec warmup 40 sec latency 60.023 ms 2024-09-13T05:27:47.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 18982 14.54 MB/sec warmup 41 sec latency 32.414 ms 2024-09-13T05:27:47.668 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:47 smithi053 ceph-mon[23721]: pgmap v261: 97 pgs: 97 active+clean; 176 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.6 MiB/s wr, 749 op/s 2024-09-13T05:27:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:47 smithi110 ceph-mon[25804]: pgmap v261: 97 pgs: 97 active+clean; 176 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.6 MiB/s wr, 749 op/s 2024-09-13T05:27:48.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 19668 14.54 MB/sec warmup 42 sec latency 36.885 ms 2024-09-13T05:27:49.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 20444 14.64 MB/sec warmup 43 sec latency 25.441 ms 2024-09-13T05:27:49.680 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:49 smithi053 ceph-mon[23721]: pgmap v262: 97 pgs: 97 active+clean; 183 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.5 MiB/s wr, 831 op/s 2024-09-13T05:27:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:49 smithi110 ceph-mon[25804]: pgmap v262: 97 pgs: 97 active+clean; 183 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 9.3 KiB/s rd, 8.5 MiB/s wr, 831 op/s 2024-09-13T05:27:50.381 INFO:teuthology.orchestra.run.smithi053.stdout: 5 21244 15.05 MB/sec warmup 44 sec latency 31.488 ms 2024-09-13T05:27:51.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 21820 15.09 MB/sec warmup 45 sec latency 26.285 ms 2024-09-13T05:27:51.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:51 smithi053 ceph-mon[23721]: pgmap v263: 97 pgs: 97 active+clean; 180 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 8.1 MiB/s wr, 745 op/s 2024-09-13T05:27:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:51 smithi110 ceph-mon[25804]: pgmap v263: 97 pgs: 97 active+clean; 180 MiB data, 690 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 8.1 MiB/s wr, 745 op/s 2024-09-13T05:27:52.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 22270 14.86 MB/sec warmup 46 sec latency 33.579 ms 2024-09-13T05:27:53.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 22546 14.72 MB/sec warmup 47 sec latency 92.546 ms 2024-09-13T05:27:53.738 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:53 smithi053 ceph-mon[23721]: pgmap v264: 97 pgs: 97 active+clean; 179 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 671 op/s 2024-09-13T05:27:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:53 smithi110 ceph-mon[25804]: pgmap v264: 97 pgs: 97 active+clean; 179 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 671 op/s 2024-09-13T05:27:54.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 23013 14.69 MB/sec warmup 48 sec latency 44.778 ms 2024-09-13T05:27:55.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 23747 14.62 MB/sec warmup 49 sec latency 24.241 ms 2024-09-13T05:27:55.783 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:55 smithi053 ceph-mon[23721]: pgmap v265: 97 pgs: 97 active+clean; 188 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 853 op/s 2024-09-13T05:27:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:55 smithi110 ceph-mon[25804]: pgmap v265: 97 pgs: 97 active+clean; 188 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 8.6 MiB/s wr, 853 op/s 2024-09-13T05:27:56.379 INFO:teuthology.orchestra.run.smithi053.stdout: 5 24605 15.00 MB/sec warmup 50 sec latency 17.357 ms 2024-09-13T05:27:57.380 INFO:teuthology.orchestra.run.smithi053.stdout: 5 25169 15.16 MB/sec warmup 51 sec latency 20.781 ms 2024-09-13T05:27:57.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:57 smithi053 ceph-mon[23721]: pgmap v266: 97 pgs: 97 active+clean; 191 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 854 op/s 2024-09-13T05:27:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:57 smithi110 ceph-mon[25804]: pgmap v266: 97 pgs: 97 active+clean; 191 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.5 MiB/s wr, 854 op/s 2024-09-13T05:27:58.384 INFO:teuthology.orchestra.run.smithi053.stdout: 5 25542 14.91 MB/sec warmup 52 sec latency 64.950 ms 2024-09-13T05:27:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:27:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:27:58.807 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:27:58.808 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:27:59.384 INFO:teuthology.orchestra.run.smithi053.stdout: 5 26171 14.92 MB/sec warmup 53 sec latency 24.121 ms 2024-09-13T05:27:59.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:27:59 smithi053 ceph-mon[23721]: pgmap v267: 97 pgs: 97 active+clean; 191 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 679 op/s 2024-09-13T05:27:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:27:59 smithi110 ceph-mon[25804]: pgmap v267: 97 pgs: 97 active+clean; 191 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 679 op/s 2024-09-13T05:28:00.384 INFO:teuthology.orchestra.run.smithi053.stdout: 5 27002 14.89 MB/sec warmup 54 sec latency 18.923 ms 2024-09-13T05:28:01.384 INFO:teuthology.orchestra.run.smithi053.stdout: 5 27818 15.14 MB/sec warmup 55 sec latency 13.397 ms 2024-09-13T05:28:01.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:01 smithi053 ceph-mon[23721]: pgmap v268: 97 pgs: 97 active+clean; 209 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 812 op/s 2024-09-13T05:28:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:01 smithi110 ceph-mon[25804]: pgmap v268: 97 pgs: 97 active+clean; 209 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.7 MiB/s wr, 812 op/s 2024-09-13T05:28:02.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 28601 15.45 MB/sec warmup 56 sec latency 26.575 ms 2024-09-13T05:28:03.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 28857 15.25 MB/sec warmup 57 sec latency 62.503 ms 2024-09-13T05:28:03.857 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:03 smithi053 ceph-mon[23721]: pgmap v269: 97 pgs: 97 active+clean; 209 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 780 op/s 2024-09-13T05:28:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:03 smithi110 ceph-mon[25804]: pgmap v269: 97 pgs: 97 active+clean; 209 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 780 op/s 2024-09-13T05:28:04.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 29251 15.06 MB/sec warmup 58 sec latency 51.955 ms 2024-09-13T05:28:05.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 29584 14.93 MB/sec warmup 59 sec latency 156.858 ms 2024-09-13T05:28:05.815 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:05 smithi053 ceph-mon[23721]: pgmap v270: 97 pgs: 97 active+clean; 209 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 832 op/s 2024-09-13T05:28:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:05 smithi110 ceph-mon[25804]: pgmap v270: 97 pgs: 97 active+clean; 209 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 9.2 MiB/s wr, 832 op/s 2024-09-13T05:28:06.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 30134 14.94 MB/sec warmup 60 sec latency 106.224 ms 2024-09-13T05:28:06.609 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:28:07.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 30682 14.78 MB/sec warmup 61 sec latency 40.424 ms 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (6m) 63s ago 7m 24.6M - ba2b418f427c 2a3ae76369c5 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (7m) 63s ago 7m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (6m) 65s ago 6m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (5m) 63s ago 7m 49.6M - 8.3.5 dad864ee21e9 41e920907455 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (95s) 63s ago 94s 14.8M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (92s) 65s ago 92s 13.4M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (8m) 63s ago 8m 460M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (6m) 65s ago 6m 411M - 17.2.0 e1d6a67b021e 69db5250e9d9 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (8m) 63s ago 8m 43.6M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:28:07.600 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (6m) 65s ago 6m 37.6M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (74s) 63s ago 73s 49.1M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (69s) 65s ago 68s 46.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (7m) 63s ago 7m 15.0M - 1dbe0e931976 aa1768a447f0 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (6m) 65s ago 6m 18.6M - 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (6m) 63s ago 6m 42.3M 3217M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (5m) 63s ago 5m 46.0M 3217M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (5m) 63s ago 5m 40.3M 3217M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (4m) 63s ago 4m 42.8M 3217M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (4m) 65s ago 4m 40.2M 3984M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:28:07.601 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (3m) 65s ago 3m 41.9M 3984M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:28:07.602 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (3m) 65s ago 3m 39.4M 3984M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:28:07.602 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (3m) 65s ago 3m 41.1M 3984M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:28:07.602 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (6m) 63s ago 6m 62.1M - 514e6a882f6e 7c997d72a66c 2024-09-13T05:28:07.758 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:07 smithi053 ceph-mon[23721]: pgmap v271: 97 pgs: 97 active+clean; 220 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 9.3 MiB/s wr, 877 op/s 2024-09-13T05:28:07.759 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:07 smithi053 ceph-mon[23721]: from='client.24433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:07 smithi110 ceph-mon[25804]: pgmap v271: 97 pgs: 97 active+clean; 220 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 9.3 MiB/s wr, 877 op/s 2024-09-13T05:28:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:07 smithi110 ceph-mon[25804]: from='client.24433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:08.095 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:28:08.095 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T05:28:08.096 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:28:08.097 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:28:08.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 31158 14.81 MB/sec warmup 62 sec latency 81.737 ms 2024-09-13T05:28:08.540 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:28:08.540 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3", 2024-09-13T05:28:08.540 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:28:08.540 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:28:08.540 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "", 2024-09-13T05:28:08.541 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 image" 2024-09-13T05:28:08.541 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:28:08.855 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:08 smithi053 ceph-mon[23721]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:08.855 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:08 smithi053 ceph-mon[23721]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:08.855 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:08 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2140033732' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:28:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:08 smithi110 ceph-mon[25804]: from='client.14656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:08 smithi110 ceph-mon[25804]: from='client.14660 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:08 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2140033732' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:28:09.039 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:28:09.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 31686 14.94 MB/sec warmup 63 sec latency 63.166 ms 2024-09-13T05:28:09.876 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:09 smithi053 ceph-mon[23721]: pgmap v272: 97 pgs: 97 active+clean; 217 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 8.7 MiB/s wr, 814 op/s 2024-09-13T05:28:09.877 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:09 smithi053 ceph-mon[23721]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:09.877 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:09 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/4056673328' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:28:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:09 smithi110 ceph-mon[25804]: pgmap v272: 97 pgs: 97 active+clean; 217 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 8.7 MiB/s wr, 814 op/s 2024-09-13T05:28:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:09 smithi110 ceph-mon[25804]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:09 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/4056673328' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:28:10.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 32137 15.01 MB/sec warmup 64 sec latency 43.667 ms 2024-09-13T05:28:11.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 32463 14.86 MB/sec warmup 65 sec latency 38.259 ms 2024-09-13T05:28:11.848 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:11 smithi053 ceph-mon[23721]: pgmap v273: 97 pgs: 97 active+clean; 215 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.6 MiB/s wr, 811 op/s 2024-09-13T05:28:11.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:11 smithi110 ceph-mon[25804]: pgmap v273: 97 pgs: 97 active+clean; 215 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.6 MiB/s wr, 811 op/s 2024-09-13T05:28:12.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 33094 14.79 MB/sec warmup 66 sec latency 32.882 ms 2024-09-13T05:28:13.114 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:28:13.115 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: Upgrade: Target is version 19.3.0-4911-g9d608eb1 (unknown) 2024-09-13T05:28:13.115 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, digests ['quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff'] 2024-09-13T05:28:13.115 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:28:13.115 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:28:13.115 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi053.ygzjut) 2024-09-13T05:28:13.115 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: pgmap v274: 97 pgs: 97 active+clean; 220 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 562 op/s 2024-09-13T05:28:13.115 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:13 smithi053 ceph-mon[23721]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff on smithi110 2024-09-13T05:28:13.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 33669 14.81 MB/sec warmup 67 sec latency 43.918 ms 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: Upgrade: Target is version 19.3.0-4911-g9d608eb1 (unknown) 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, digests ['quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff'] 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: Upgrade: Need to upgrade myself (mgr.smithi053.ygzjut) 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: pgmap v274: 97 pgs: 97 active+clean; 220 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 562 op/s 2024-09-13T05:28:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:13 smithi110 ceph-mon[25804]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff on smithi110 2024-09-13T05:28:14.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 34265 14.68 MB/sec warmup 68 sec latency 49.506 ms 2024-09-13T05:28:15.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 34662 14.68 MB/sec warmup 69 sec latency 39.960 ms 2024-09-13T05:28:15.647 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:15 smithi053 ceph-mon[23721]: pgmap v275: 97 pgs: 97 active+clean; 227 MiB data, 1013 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 674 op/s 2024-09-13T05:28:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:15 smithi110 ceph-mon[25804]: pgmap v275: 97 pgs: 97 active+clean; 227 MiB data, 1013 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.5 MiB/s wr, 674 op/s 2024-09-13T05:28:16.385 INFO:teuthology.orchestra.run.smithi053.stdout: 5 35103 14.77 MB/sec warmup 70 sec latency 58.776 ms 2024-09-13T05:28:17.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 35507 14.77 MB/sec warmup 71 sec latency 62.475 ms 2024-09-13T05:28:17.687 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:17 smithi053 ceph-mon[23721]: pgmap v276: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 820 op/s 2024-09-13T05:28:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:17 smithi110 ceph-mon[25804]: pgmap v276: 97 pgs: 97 active+clean; 231 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.7 MiB/s wr, 820 op/s 2024-09-13T05:28:18.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 36041 14.74 MB/sec warmup 72 sec latency 64.786 ms 2024-09-13T05:28:19.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 36453 14.61 MB/sec warmup 73 sec latency 64.946 ms 2024-09-13T05:28:19.708 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:19 smithi053 ceph-mon[23721]: pgmap v277: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 627 op/s 2024-09-13T05:28:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:19 smithi110 ceph-mon[25804]: pgmap v277: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.5 MiB/s wr, 627 op/s 2024-09-13T05:28:20.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 36855 14.55 MB/sec warmup 74 sec latency 72.686 ms 2024-09-13T05:28:21.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 37315 14.48 MB/sec warmup 75 sec latency 43.353 ms 2024-09-13T05:28:21.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:21 smithi053 ceph-mon[23721]: pgmap v278: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-09-13T05:28:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:21 smithi110 ceph-mon[25804]: pgmap v278: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-09-13T05:28:22.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 38006 14.51 MB/sec warmup 76 sec latency 72.587 ms 2024-09-13T05:28:23.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 38752 14.72 MB/sec warmup 77 sec latency 70.922 ms 2024-09-13T05:28:23.775 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:23 smithi053 ceph-mon[23721]: pgmap v279: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.0 MiB/s wr, 693 op/s 2024-09-13T05:28:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:23 smithi110 ceph-mon[25804]: pgmap v279: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.0 MiB/s wr, 693 op/s 2024-09-13T05:28:24.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 39376 14.80 MB/sec warmup 78 sec latency 15.559 ms 2024-09-13T05:28:25.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 40045 14.71 MB/sec warmup 79 sec latency 21.561 ms 2024-09-13T05:28:25.786 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:25 smithi053 ceph-mon[23721]: pgmap v280: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 805 op/s 2024-09-13T05:28:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:25 smithi110 ceph-mon[25804]: pgmap v280: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 805 op/s 2024-09-13T05:28:26.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 41019 14.82 MB/sec warmup 80 sec latency 21.624 ms 2024-09-13T05:28:27.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 41764 14.87 MB/sec warmup 81 sec latency 26.358 ms 2024-09-13T05:28:27.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:27 smithi053 ceph-mon[23721]: pgmap v281: 97 pgs: 97 active+clean; 251 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.0 MiB/s wr, 835 op/s 2024-09-13T05:28:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:27 smithi110 ceph-mon[25804]: pgmap v281: 97 pgs: 97 active+clean; 251 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.0 MiB/s wr, 835 op/s 2024-09-13T05:28:28.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 42207 14.97 MB/sec warmup 82 sec latency 41.365 ms 2024-09-13T05:28:29.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 42485 14.87 MB/sec warmup 83 sec latency 57.907 ms 2024-09-13T05:28:29.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:29 smithi053 ceph-mon[23721]: pgmap v282: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.0 MiB/s wr, 693 op/s 2024-09-13T05:28:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:29 smithi110 ceph-mon[25804]: pgmap v282: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.0 MiB/s wr, 693 op/s 2024-09-13T05:28:30.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 42797 14.86 MB/sec warmup 84 sec latency 58.906 ms 2024-09-13T05:28:31.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 43015 14.72 MB/sec warmup 85 sec latency 54.866 ms 2024-09-13T05:28:31.692 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:31 smithi053 ceph-mon[23721]: pgmap v283: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 891 op/s 2024-09-13T05:28:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:31 smithi110 ceph-mon[25804]: pgmap v283: 97 pgs: 97 active+clean; 259 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 891 op/s 2024-09-13T05:28:32.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 43339 14.58 MB/sec warmup 86 sec latency 143.225 ms 2024-09-13T05:28:33.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 43560 14.46 MB/sec warmup 87 sec latency 627.748 ms 2024-09-13T05:28:33.855 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:33 smithi053 ceph-mon[23721]: pgmap v284: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 691 op/s 2024-09-13T05:28:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:33 smithi110 ceph-mon[25804]: pgmap v284: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 691 op/s 2024-09-13T05:28:34.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 43843 14.40 MB/sec warmup 88 sec latency 45.811 ms 2024-09-13T05:28:35.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 44207 14.33 MB/sec warmup 89 sec latency 67.093 ms 2024-09-13T05:28:35.663 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:35 smithi053 ceph-mon[23721]: pgmap v285: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-09-13T05:28:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:35 smithi110 ceph-mon[25804]: pgmap v285: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 769 op/s 2024-09-13T05:28:36.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 44681 14.25 MB/sec warmup 90 sec latency 74.738 ms 2024-09-13T05:28:37.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 45337 14.35 MB/sec warmup 91 sec latency 40.038 ms 2024-09-13T05:28:37.691 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:37 smithi053 ceph-mon[23721]: pgmap v286: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 637 op/s 2024-09-13T05:28:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:37 smithi110 ceph-mon[25804]: pgmap v286: 97 pgs: 97 active+clean; 266 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 637 op/s 2024-09-13T05:28:38.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 46254 14.56 MB/sec warmup 92 sec latency 20.256 ms 2024-09-13T05:28:39.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 46588 14.49 MB/sec warmup 93 sec latency 51.951 ms 2024-09-13T05:28:39.575 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:28:39.728 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:39 smithi053 ceph-mon[23721]: pgmap v287: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 602 op/s 2024-09-13T05:28:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:39 smithi110 ceph-mon[25804]: pgmap v287: 97 pgs: 97 active+clean; 264 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 602 op/s 2024-09-13T05:28:40.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 46953 14.37 MB/sec warmup 94 sec latency 40.742 ms 2024-09-13T05:28:40.415 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:28:40.415 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (6m) 96s ago 8m 24.6M - ba2b418f427c 2a3ae76369c5 2024-09-13T05:28:40.415 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (8m) 96s ago 8m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:28:40.415 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (7m) 98s ago 7m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:28:40.415 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (6m) 96s ago 7m 49.6M - 8.3.5 dad864ee21e9 41e920907455 2024-09-13T05:28:40.415 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (2m) 96s ago 2m 14.8M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (2m) 98s ago 2m 13.4M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (9m) 96s ago 9m 460M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (7m) 98s ago 7m 411M - 17.2.0 e1d6a67b021e 69db5250e9d9 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (9m) 96s ago 9m 43.6M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (7m) 98s ago 7m 37.6M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (107s) 96s ago 106s 49.1M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (101s) 98s ago 101s 46.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (7m) 96s ago 7m 15.0M - 1dbe0e931976 aa1768a447f0 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (7m) 98s ago 7m 18.6M - 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (6m) 96s ago 6m 42.3M 3217M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (6m) 96s ago 6m 46.0M 3217M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (5m) 96s ago 5m 40.3M 3217M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:28:40.416 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (5m) 96s ago 5m 42.8M 3217M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:28:40.417 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (4m) 98s ago 4m 40.2M 3984M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:28:40.417 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (4m) 98s ago 4m 41.9M 3984M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:28:40.417 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (4m) 98s ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:28:40.417 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (3m) 98s ago 3m 41.1M 3984M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:28:40.417 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (7m) 96s ago 7m 62.1M - 514e6a882f6e 7c997d72a66c 2024-09-13T05:28:40.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:40 smithi053 ceph-mon[23721]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:40.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:40 smithi053 ceph-mon[23721]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:40.898 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:28:40.898 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:28:40.898 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:28:40.898 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:40.898 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:28:40.898 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:28:40.899 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:28:40.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:40 smithi110 ceph-mon[25804]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:40.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:40 smithi110 ceph-mon[25804]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:41.328 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:28:41.328 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:28:41.329 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:28:41.329 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:28:41.329 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "0/23 daemons upgraded", 2024-09-13T05:28:41.329 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff image on host smithi110" 2024-09-13T05:28:41.329 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:28:41.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 47423 14.35 MB/sec warmup 95 sec latency 85.063 ms 2024-09-13T05:28:41.767 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:41 smithi053 ceph-mon[23721]: pgmap v288: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 742 op/s 2024-09-13T05:28:41.767 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:41 smithi053 ceph-mon[23721]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:41.767 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:41 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1355564809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:28:41.845 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:28:41.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:41 smithi110 ceph-mon[25804]: pgmap v288: 97 pgs: 97 active+clean; 276 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 742 op/s 2024-09-13T05:28:41.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:41 smithi110 ceph-mon[25804]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:41.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:41 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1355564809' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:28:42.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 48074 14.33 MB/sec warmup 96 sec latency 37.997 ms 2024-09-13T05:28:42.774 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:42 smithi053 ceph-mon[23721]: from='client.14692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:42.775 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:42 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3166448480' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:28:42.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:42 smithi110 ceph-mon[25804]: from='client.14692 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:28:42.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:42 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3166448480' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:28:43.386 INFO:teuthology.orchestra.run.smithi053.stdout: 5 48507 14.25 MB/sec warmup 97 sec latency 38.791 ms 2024-09-13T05:28:43.800 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:43 smithi053 ceph-mon[23721]: pgmap v289: 97 pgs: 97 active+clean; 277 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 505 op/s 2024-09-13T05:28:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:43 smithi110 ceph-mon[25804]: pgmap v289: 97 pgs: 97 active+clean; 277 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 505 op/s 2024-09-13T05:28:44.387 INFO:teuthology.orchestra.run.smithi053.stdout: 5 49373 14.46 MB/sec warmup 98 sec latency 21.435 ms 2024-09-13T05:28:45.387 INFO:teuthology.orchestra.run.smithi053.stdout: 5 49938 14.55 MB/sec warmup 99 sec latency 40.532 ms 2024-09-13T05:28:45.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:45 smithi053 ceph-mon[23721]: pgmap v290: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 686 op/s 2024-09-13T05:28:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:45 smithi110 ceph-mon[25804]: pgmap v290: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 686 op/s 2024-09-13T05:28:46.387 INFO:teuthology.orchestra.run.smithi053.stdout: 5 50284 14.44 MB/sec warmup 100 sec latency 44.591 ms 2024-09-13T05:28:47.387 INFO:teuthology.orchestra.run.smithi053.stdout: 5 50607 14.34 MB/sec warmup 101 sec latency 63.325 ms 2024-09-13T05:28:47.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:47 smithi053 ceph-mon[23721]: pgmap v291: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.5 MiB/s wr, 652 op/s 2024-09-13T05:28:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:47 smithi110 ceph-mon[25804]: pgmap v291: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.5 MiB/s wr, 652 op/s 2024-09-13T05:28:48.387 INFO:teuthology.orchestra.run.smithi053.stdout: 5 50929 14.30 MB/sec warmup 102 sec latency 45.782 ms 2024-09-13T05:28:49.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 51626 14.28 MB/sec warmup 103 sec latency 38.982 ms 2024-09-13T05:28:49.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:49 smithi053 ceph-mon[23721]: pgmap v292: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 683 op/s 2024-09-13T05:28:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:49 smithi110 ceph-mon[25804]: pgmap v292: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 7.5 MiB/s wr, 683 op/s 2024-09-13T05:28:50.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 52042 14.20 MB/sec warmup 104 sec latency 50.432 ms 2024-09-13T05:28:51.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 52377 14.19 MB/sec warmup 105 sec latency 93.521 ms 2024-09-13T05:28:51.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:51 smithi053 ceph-mon[23721]: pgmap v293: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 781 op/s 2024-09-13T05:28:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:51 smithi110 ceph-mon[25804]: pgmap v293: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.0 MiB/s wr, 781 op/s 2024-09-13T05:28:52.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 52921 14.27 MB/sec warmup 106 sec latency 71.707 ms 2024-09-13T05:28:53.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 53410 14.35 MB/sec warmup 107 sec latency 75.540 ms 2024-09-13T05:28:53.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:53 smithi053 ceph-mon[23721]: pgmap v294: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 673 op/s 2024-09-13T05:28:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:53 smithi110 ceph-mon[25804]: pgmap v294: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.7 MiB/s wr, 673 op/s 2024-09-13T05:28:54.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 53695 14.25 MB/sec warmup 108 sec latency 53.607 ms 2024-09-13T05:28:55.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 53954 14.13 MB/sec warmup 109 sec latency 91.659 ms 2024-09-13T05:28:55.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:55 smithi053 ceph-mon[23721]: pgmap v295: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 708 op/s 2024-09-13T05:28:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:55 smithi110 ceph-mon[25804]: pgmap v295: 97 pgs: 97 active+clean; 293 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 7.3 MiB/s wr, 708 op/s 2024-09-13T05:28:56.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 54204 14.04 MB/sec warmup 110 sec latency 74.835 ms 2024-09-13T05:28:57.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 54566 14.04 MB/sec warmup 111 sec latency 41.155 ms 2024-09-13T05:28:57.844 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:57 smithi053 ceph-mon[23721]: pgmap v296: 97 pgs: 97 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.1 MiB/s wr, 618 op/s 2024-09-13T05:28:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:57 smithi110 ceph-mon[25804]: pgmap v296: 97 pgs: 97 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.1 MiB/s wr, 618 op/s 2024-09-13T05:28:58.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 55038 13.98 MB/sec warmup 112 sec latency 41.577 ms 2024-09-13T05:28:58.857 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:28:58.857 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:58 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:28:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:28:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:58 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:28:59.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 55468 13.90 MB/sec warmup 113 sec latency 78.040 ms 2024-09-13T05:28:59.884 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:28:59 smithi053 ceph-mon[23721]: pgmap v297: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.2 MiB/s wr, 672 op/s 2024-09-13T05:28:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:28:59 smithi110 ceph-mon[25804]: pgmap v297: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.2 MiB/s wr, 672 op/s 2024-09-13T05:29:00.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 56138 14.01 MB/sec warmup 114 sec latency 31.836 ms 2024-09-13T05:29:01.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 56839 14.12 MB/sec warmup 115 sec latency 30.595 ms 2024-09-13T05:29:01.929 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:01 smithi053 ceph-mon[23721]: pgmap v298: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 728 op/s 2024-09-13T05:29:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:01 smithi110 ceph-mon[25804]: pgmap v298: 97 pgs: 97 active+clean; 299 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 728 op/s 2024-09-13T05:29:02.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 57069 14.07 MB/sec warmup 116 sec latency 55.821 ms 2024-09-13T05:29:03.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 57303 13.97 MB/sec warmup 117 sec latency 64.807 ms 2024-09-13T05:29:03.961 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:03 smithi053 ceph-mon[23721]: pgmap v299: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.4 MiB/s wr, 519 op/s 2024-09-13T05:29:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:03 smithi110 ceph-mon[25804]: pgmap v299: 97 pgs: 97 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.4 MiB/s wr, 519 op/s 2024-09-13T05:29:04.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 57921 13.93 MB/sec warmup 118 sec latency 55.673 ms 2024-09-13T05:29:05.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 58733 13.97 MB/sec warmup 119 sec latency 28.711 ms 2024-09-13T05:29:05.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:05 smithi053 ceph-mon[23721]: pgmap v300: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 459 op/s 2024-09-13T05:29:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:05 smithi110 ceph-mon[25804]: pgmap v300: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.7 MiB/s wr, 459 op/s 2024-09-13T05:29:07.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 59577 19.18 MB/sec execute 1 sec latency 50.380 ms 2024-09-13T05:29:07.806 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:07 smithi053 ceph-mon[23721]: pgmap v301: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 558 op/s 2024-09-13T05:29:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:07 smithi110 ceph-mon[25804]: pgmap v301: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 558 op/s 2024-09-13T05:29:08.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 60024 17.23 MB/sec execute 2 sec latency 70.859 ms 2024-09-13T05:29:09.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 60536 19.23 MB/sec execute 3 sec latency 50.632 ms 2024-09-13T05:29:09.833 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:09 smithi053 ceph-mon[23721]: pgmap v302: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.6 MiB/s wr, 672 op/s 2024-09-13T05:29:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:09 smithi110 ceph-mon[25804]: pgmap v302: 97 pgs: 97 active+clean; 321 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.6 MiB/s wr, 672 op/s 2024-09-13T05:29:10.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 61034 15.54 MB/sec execute 4 sec latency 30.903 ms 2024-09-13T05:29:11.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 61596 15.42 MB/sec execute 5 sec latency 24.558 ms 2024-09-13T05:29:11.852 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:11 smithi053 ceph-mon[23721]: pgmap v303: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 613 op/s 2024-09-13T05:29:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:11 smithi110 ceph-mon[25804]: pgmap v303: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 613 op/s 2024-09-13T05:29:12.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 62244 14.71 MB/sec execute 6 sec latency 40.258 ms 2024-09-13T05:29:12.410 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:29:13.394 INFO:teuthology.orchestra.run.smithi053.stdout: 5 62778 13.95 MB/sec execute 7 sec latency 30.384 ms 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (7m) 2m ago 8m 24.6M - ba2b418f427c 2a3ae76369c5 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (8m) 2m ago 8m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (8m) 2m ago 8m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (6m) 2m ago 8m 49.6M - 8.3.5 dad864ee21e9 41e920907455 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (2m) 2m ago 2m 14.8M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (2m) 2m ago 2m 13.4M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:29:13.438 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (7m) 2m ago 7m 411M - 17.2.0 e1d6a67b021e 69db5250e9d9 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (9m) 2m ago 10m 43.6M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (7m) 2m ago 7m 37.6M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (2m) 2m ago 2m 49.1M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (2m) 2m ago 2m 46.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (8m) 2m ago 8m 15.0M - 1dbe0e931976 aa1768a447f0 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (7m) 2m ago 7m 18.6M - 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (7m) 2m ago 7m 42.3M 3217M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (6m) 2m ago 6m 46.0M 3217M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (6m) 2m ago 6m 40.3M 3217M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (5m) 2m ago 5m 42.8M 3217M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (5m) 2m ago 5m 40.2M 3984M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:29:13.439 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (4m) 2m ago 4m 41.9M 3984M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:29:13.440 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (4m) 2m ago 4m 39.4M 3984M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:29:13.440 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (4m) 2m ago 4m 41.1M 3984M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:29:13.440 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (7m) 2m ago 7m 62.1M - 514e6a882f6e 7c997d72a66c 2024-09-13T05:29:13.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:13 smithi053 ceph-mon[23721]: from='client.24463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:13.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:13 smithi053 ceph-mon[23721]: pgmap v304: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.7 MiB/s wr, 610 op/s 2024-09-13T05:29:13.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:13 smithi053 ceph-mon[23721]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:29:14.074 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:14.075 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:29:14.075 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:29:14.075 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:14.075 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:29:14.075 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T05:29:14.075 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:29:14.075 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:29:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:13 smithi110 ceph-mon[25804]: from='client.24463 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:13 smithi110 ceph-mon[25804]: pgmap v304: 97 pgs: 97 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.7 MiB/s wr, 610 op/s 2024-09-13T05:29:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:13 smithi110 ceph-mon[25804]: from='client.14704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:14.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 63417 15.82 MB/sec execute 8 sec latency 22.406 ms 2024-09-13T05:29:14.579 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:29:14.579 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:29:14.580 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:29:14.580 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:29:14.580 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "0/23 daemons upgraded", 2024-09-13T05:29:14.580 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff image on host smithi110" 2024-09-13T05:29:14.580 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:29:14.891 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:14 smithi053 ceph-mon[23721]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:14.892 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:14 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/409782889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:29:15.143 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:29:15.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:14 smithi110 ceph-mon[25804]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:15.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:14 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/409782889' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:29:15.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 63894 16.20 MB/sec execute 9 sec latency 40.642 ms 2024-09-13T05:29:15.914 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:15 smithi053 ceph-mon[23721]: pgmap v305: 97 pgs: 97 active+clean; 327 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 708 op/s 2024-09-13T05:29:15.914 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:15 smithi053 ceph-mon[23721]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:15.914 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:15 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1699099849' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:29:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:15 smithi110 ceph-mon[25804]: pgmap v305: 97 pgs: 97 active+clean; 327 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 708 op/s 2024-09-13T05:29:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:15 smithi110 ceph-mon[25804]: from='client.14716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:15 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1699099849' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:29:16.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 64155 15.51 MB/sec execute 10 sec latency 41.510 ms 2024-09-13T05:29:17.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 64465 14.27 MB/sec execute 11 sec latency 43.879 ms 2024-09-13T05:29:18.032 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:17 smithi053 ceph-mon[23721]: pgmap v306: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.0 MiB/s wr, 867 op/s 2024-09-13T05:29:18.045 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:17 smithi110 ceph-mon[25804]: pgmap v306: 97 pgs: 97 active+clean; 331 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 8.0 MiB/s wr, 867 op/s 2024-09-13T05:29:18.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 64723 13.29 MB/sec execute 12 sec latency 101.109 ms 2024-09-13T05:29:19.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 64905 12.62 MB/sec execute 13 sec latency 103.534 ms 2024-09-13T05:29:19.927 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:19 smithi053 ceph-mon[23721]: pgmap v307: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 768 op/s 2024-09-13T05:29:20.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:19 smithi110 ceph-mon[25804]: pgmap v307: 97 pgs: 97 active+clean; 337 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.8 MiB/s wr, 768 op/s 2024-09-13T05:29:20.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 65202 12.55 MB/sec execute 14 sec latency 49.020 ms 2024-09-13T05:29:21.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 65540 12.18 MB/sec execute 15 sec latency 52.700 ms 2024-09-13T05:29:21.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:21 smithi053 ceph-mon[23721]: pgmap v308: 97 pgs: 97 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 735 op/s 2024-09-13T05:29:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:21 smithi110 ceph-mon[25804]: pgmap v308: 97 pgs: 97 active+clean; 344 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.5 MiB/s wr, 735 op/s 2024-09-13T05:29:22.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 65958 11.58 MB/sec execute 16 sec latency 47.267 ms 2024-09-13T05:29:23.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 66675 12.43 MB/sec execute 17 sec latency 22.080 ms 2024-09-13T05:29:23.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:23 smithi053 ceph-mon[23721]: pgmap v309: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 700 op/s 2024-09-13T05:29:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:23 smithi110 ceph-mon[25804]: pgmap v309: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.4 MiB/s wr, 700 op/s 2024-09-13T05:29:24.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 67491 13.50 MB/sec execute 18 sec latency 28.587 ms 2024-09-13T05:29:25.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 67798 13.25 MB/sec execute 19 sec latency 64.103 ms 2024-09-13T05:29:25.964 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:25 smithi053 ceph-mon[23721]: pgmap v310: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 725 op/s 2024-09-13T05:29:26.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:25 smithi110 ceph-mon[25804]: pgmap v310: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 725 op/s 2024-09-13T05:29:26.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 68173 12.71 MB/sec execute 20 sec latency 64.022 ms 2024-09-13T05:29:27.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 68484 12.41 MB/sec execute 21 sec latency 213.995 ms 2024-09-13T05:29:28.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:27 smithi053 ceph-mon[23721]: pgmap v311: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 739 op/s 2024-09-13T05:29:28.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:27 smithi110 ceph-mon[25804]: pgmap v311: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.4 MiB/s wr, 739 op/s 2024-09-13T05:29:28.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 68994 12.51 MB/sec execute 22 sec latency 53.911 ms 2024-09-13T05:29:29.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 69601 12.27 MB/sec execute 23 sec latency 39.886 ms 2024-09-13T05:29:29.783 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:29 smithi053 ceph-mon[23721]: pgmap v312: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.3 MiB/s wr, 587 op/s 2024-09-13T05:29:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:29 smithi110 ceph-mon[25804]: pgmap v312: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 6.3 MiB/s wr, 587 op/s 2024-09-13T05:29:30.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 69976 12.29 MB/sec execute 24 sec latency 24.195 ms 2024-09-13T05:29:31.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 70453 12.62 MB/sec execute 25 sec latency 28.081 ms 2024-09-13T05:29:31.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:31 smithi053 ceph-mon[23721]: pgmap v313: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 740 op/s 2024-09-13T05:29:32.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:31 smithi110 ceph-mon[25804]: pgmap v313: 97 pgs: 97 active+clean; 337 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 740 op/s 2024-09-13T05:29:32.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 71087 13.05 MB/sec execute 26 sec latency 29.394 ms 2024-09-13T05:29:33.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 71532 12.91 MB/sec execute 27 sec latency 32.857 ms 2024-09-13T05:29:33.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:33 smithi053 ceph-mon[23721]: pgmap v314: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-09-13T05:29:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:33 smithi110 ceph-mon[25804]: pgmap v314: 97 pgs: 97 active+clean; 335 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 566 op/s 2024-09-13T05:29:34.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 71878 12.57 MB/sec execute 28 sec latency 37.364 ms 2024-09-13T05:29:35.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 72298 12.46 MB/sec execute 29 sec latency 34.706 ms 2024-09-13T05:29:35.856 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:35 smithi053 ceph-mon[23721]: pgmap v315: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 760 op/s 2024-09-13T05:29:36.098 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:35 smithi110 ceph-mon[25804]: pgmap v315: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 760 op/s 2024-09-13T05:29:36.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 73087 12.60 MB/sec execute 30 sec latency 28.529 ms 2024-09-13T05:29:37.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 73956 13.20 MB/sec execute 31 sec latency 22.780 ms 2024-09-13T05:29:37.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:37 smithi053 ceph-mon[23721]: pgmap v316: 97 pgs: 97 active+clean; 343 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.4 MiB/s wr, 713 op/s 2024-09-13T05:29:37.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:37 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:37.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:37 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:29:37.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:37 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:29:37.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:37 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:29:37.908 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:37 smithi110 ceph-mon[25804]: pgmap v316: 97 pgs: 97 active+clean; 343 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.4 MiB/s wr, 713 op/s 2024-09-13T05:29:37.909 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:37 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:37.909 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:37 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:29:37.909 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:37 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:29:37.909 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:37 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:29:38.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 74834 13.79 MB/sec execute 32 sec latency 18.022 ms 2024-09-13T05:29:38.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:38 smithi053 ceph-mon[23721]: Upgrade: Updating mgr.smithi110.rivldx 2024-09-13T05:29:38.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:38 smithi053 ceph-mon[23721]: Deploying daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:29:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:38 smithi110 ceph-mon[25804]: Upgrade: Updating mgr.smithi110.rivldx 2024-09-13T05:29:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:38 smithi110 ceph-mon[25804]: Deploying daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:29:39.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 75388 13.55 MB/sec execute 33 sec latency 62.454 ms 2024-09-13T05:29:39.874 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:39 smithi053 ceph-mon[23721]: pgmap v317: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.3 MiB/s wr, 645 op/s 2024-09-13T05:29:39.967 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:39 smithi110 ceph-mon[25804]: pgmap v317: 97 pgs: 97 active+clean; 336 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 7.3 MiB/s wr, 645 op/s 2024-09-13T05:29:40.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 76002 13.55 MB/sec execute 34 sec latency 32.021 ms 2024-09-13T05:29:41.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 76685 13.55 MB/sec execute 35 sec latency 43.272 ms 2024-09-13T05:29:41.899 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:41 smithi053 ceph-mon[23721]: pgmap v318: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 917 op/s 2024-09-13T05:29:41.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:41 smithi110 ceph-mon[25804]: pgmap v318: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.6 MiB/s wr, 917 op/s 2024-09-13T05:29:42.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 77165 13.53 MB/sec execute 36 sec latency 29.540 ms 2024-09-13T05:29:43.176 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:43.177 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:29:43.177 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:29:43.177 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:43 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:29:43.177 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:43 smithi053 ceph-mon[23721]: pgmap v319: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 740 op/s 2024-09-13T05:29:43.395 INFO:teuthology.orchestra.run.smithi053.stdout: 5 77547 13.66 MB/sec execute 37 sec latency 38.298 ms 2024-09-13T05:29:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:29:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:29:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:43 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:29:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:43 smithi110 ceph-mon[25804]: pgmap v319: 97 pgs: 97 active+clean; 343 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 7.6 MiB/s wr, 740 op/s 2024-09-13T05:29:44.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 77843 13.58 MB/sec execute 38 sec latency 127.552 ms 2024-09-13T05:29:45.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 78161 13.58 MB/sec execute 39 sec latency 62.541 ms 2024-09-13T05:29:45.540 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:29:45.610 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:45 smithi110 ceph-mon[25804]: pgmap v320: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 783 op/s 2024-09-13T05:29:45.790 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:45 smithi053 ceph-mon[23721]: pgmap v320: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 783 op/s 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (7m) 2m ago 9m 24.6M - ba2b418f427c 2a3ae76369c5 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (9m) 2m ago 9m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (8m) 2m ago 8m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (7m) 2m ago 8m 49.6M - 8.3.5 dad864ee21e9 41e920907455 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (3m) 2m ago 3m 14.8M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (3m) 2m ago 3m 13.4M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (10m) 2m ago 10m 460M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 starting - - - - 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (10m) 2m ago 10m 43.6M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (8m) 2m ago 8m 37.6M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:29:46.318 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (2m) 2m ago 2m 49.1M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (2m) 2m ago 2m 46.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (8m) 2m ago 8m 15.0M - 1dbe0e931976 aa1768a447f0 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (8m) 2m ago 8m 18.6M - 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (7m) 2m ago 7m 42.3M 3217M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (7m) 2m ago 7m 46.0M 3217M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (6m) 2m ago 6m 40.3M 3217M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (6m) 2m ago 6m 42.8M 3217M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (5m) 2m ago 5m 40.2M 3984M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (5m) 2m ago 5m 41.9M 3984M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (5m) 2m ago 5m 39.4M 3984M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (4m) 2m ago 4m 41.1M 3984M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:29:46.319 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (8m) 2m ago 8m 62.1M - 514e6a882f6e 7c997d72a66c 2024-09-13T05:29:46.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 78380 13.42 MB/sec execute 40 sec latency 81.237 ms 2024-09-13T05:29:46.823 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:46 smithi053 ceph-mon[23721]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:46.824 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:46 smithi053 ceph-mon[23721]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:46.841 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:29:46.841 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:29:46.841 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:29:46.841 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:46.841 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:29:46.841 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:29:46.842 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:29:46.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:46 smithi110 ceph-mon[25804]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:46.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:46 smithi110 ceph-mon[25804]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:47.241 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:29:47.241 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:29:47.241 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:29:47.241 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:29:47.241 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "", 2024-09-13T05:29:47.241 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading mgr daemons" 2024-09-13T05:29:47.241 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:29:47.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 78579 13.16 MB/sec execute 41 sec latency 56.572 ms 2024-09-13T05:29:47.764 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:29:47.857 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:47 smithi053 ceph-mon[23721]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:47.857 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:47 smithi053 ceph-mon[23721]: pgmap v321: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 744 op/s 2024-09-13T05:29:47.857 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:47 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1960738956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:29:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:47 smithi110 ceph-mon[25804]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:47 smithi110 ceph-mon[25804]: pgmap v321: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 744 op/s 2024-09-13T05:29:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:47 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1960738956' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:29:48.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 78872 12.92 MB/sec execute 42 sec latency 49.052 ms 2024-09-13T05:29:48.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:48.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1059600615' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:29:48.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: Standby manager daemon smithi110.rivldx restarted 2024-09-13T05:29:48.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:29:48.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:29:48.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:29:48.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:29:48.867 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:48 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: from='client.14740 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1059600615' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx restarted 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:29:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:48 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:29:49.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 79193 12.77 MB/sec execute 43 sec latency 73.451 ms 2024-09-13T05:29:49.639 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:49 smithi053 ceph-mon[23721]: pgmap v322: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 685 op/s 2024-09-13T05:29:49.640 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:49 smithi053 ceph-mon[23721]: mgrmap e22: smithi053.ygzjut(active, since 8m), standbys: smithi110.rivldx 2024-09-13T05:29:49.640 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:49 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:49.741 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:49 smithi110 ceph-mon[25804]: pgmap v322: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 685 op/s 2024-09-13T05:29:49.742 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:49 smithi110 ceph-mon[25804]: mgrmap e22: smithi053.ygzjut(active, since 8m), standbys: smithi110.rivldx 2024-09-13T05:29:49.742 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:49 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:50.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 79583 12.70 MB/sec execute 44 sec latency 32.453 ms 2024-09-13T05:29:51.170 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:51 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:51.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 80248 12.74 MB/sec execute 45 sec latency 38.181 ms 2024-09-13T05:29:51.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:51 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:29:52.174 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:52 smithi053 ceph-mon[23721]: pgmap v323: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.6 MiB/s wr, 717 op/s 2024-09-13T05:29:52.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 80822 12.77 MB/sec execute 46 sec latency 66.894 ms 2024-09-13T05:29:52.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:52 smithi110 ceph-mon[25804]: pgmap v323: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 7.6 MiB/s wr, 717 op/s 2024-09-13T05:29:53.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 81457 13.02 MB/sec execute 47 sec latency 43.501 ms 2024-09-13T05:29:54.189 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:54 smithi053 ceph-mon[23721]: pgmap v324: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 492 op/s 2024-09-13T05:29:54.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 82018 13.22 MB/sec execute 48 sec latency 44.231 ms 2024-09-13T05:29:54.484 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:54 smithi110 ceph-mon[25804]: pgmap v324: 97 pgs: 97 active+clean; 327 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 492 op/s 2024-09-13T05:29:55.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 82322 13.06 MB/sec execute 49 sec latency 48.439 ms 2024-09-13T05:29:56.277 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:56 smithi053 ceph-mon[23721]: pgmap v325: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-09-13T05:29:56.396 INFO:teuthology.orchestra.run.smithi053.stdout: 5 82556 12.82 MB/sec execute 50 sec latency 75.004 ms 2024-09-13T05:29:56.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:56 smithi110 ceph-mon[25804]: pgmap v325: 97 pgs: 97 active+clean; 324 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 6.1 MiB/s wr, 559 op/s 2024-09-13T05:29:57.401 INFO:teuthology.orchestra.run.smithi053.stdout: 5 83088 12.76 MB/sec execute 51 sec latency 48.389 ms 2024-09-13T05:29:58.286 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:58 smithi053 ceph-mon[23721]: pgmap v326: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.3 MiB/s wr, 641 op/s 2024-09-13T05:29:58.396 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:58 smithi110 ceph-mon[25804]: pgmap v326: 97 pgs: 97 active+clean; 329 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.3 MiB/s wr, 641 op/s 2024-09-13T05:29:58.401 INFO:teuthology.orchestra.run.smithi053.stdout: 5 83990 12.93 MB/sec execute 52 sec latency 21.323 ms 2024-09-13T05:29:59.293 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:29:59.293 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:29:59 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:29:59.401 INFO:teuthology.orchestra.run.smithi053.stdout: 5 84611 13.09 MB/sec execute 53 sec latency 28.086 ms 2024-09-13T05:29:59.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:29:59.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:29:59 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:30:00.298 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:00 smithi053 ceph-mon[23721]: pgmap v327: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.4 MiB/s wr, 583 op/s 2024-09-13T05:30:00.299 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:00 smithi053 ceph-mon[23721]: overall HEALTH_OK 2024-09-13T05:30:00.347 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:00 smithi110 ceph-mon[25804]: pgmap v327: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.4 MiB/s wr, 583 op/s 2024-09-13T05:30:00.348 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:00 smithi110 ceph-mon[25804]: overall HEALTH_OK 2024-09-13T05:30:00.406 INFO:teuthology.orchestra.run.smithi053.stdout: 5 85210 13.22 MB/sec execute 54 sec latency 54.371 ms 2024-09-13T05:30:01.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 85636 13.29 MB/sec execute 55 sec latency 23.234 ms 2024-09-13T05:30:02.316 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: pgmap v328: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 651 op/s 2024-09-13T05:30:02.316 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:30:02.316 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:30:02.316 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:30:02.316 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:30:02.317 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi053.ygzjut) 2024-09-13T05:30:02.317 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: Failing over to other MGR 2024-09-13T05:30:02.317 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr fail", "who": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:30:02.317 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:02 smithi053 ceph-mon[23721]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T05:30:02.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 86059 13.15 MB/sec execute 56 sec latency 32.957 ms 2024-09-13T05:30:02.407 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: pgmap v328: 97 pgs: 97 active+clean; 319 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 651 op/s 2024-09-13T05:30:02.407 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:30:02.407 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:30:02.407 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' 2024-09-13T05:30:02.407 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:30:02.408 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: Upgrade: Need to upgrade myself (mgr.smithi053.ygzjut) 2024-09-13T05:30:02.408 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: Failing over to other MGR 2024-09-13T05:30:02.408 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr fail", "who": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:30:02.408 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:02 smithi110 ceph-mon[25804]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "mgr fail", "who": "smithi053.ygzjut"}]': finished 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: mgrmap e23: smithi110.rivldx(active, starting, since 0.993349s) 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"}]: dispatch 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"}]: dispatch 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"}]: dispatch 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:30:03.327 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:30:03.328 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:03 smithi053 ceph-mon[23721]: Manager daemon smithi110.rivldx is now available 2024-09-13T05:30:03.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 86374 13.00 MB/sec execute 57 sec latency 56.432 ms 2024-09-13T05:30:03.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.14202 172.21.15.53:0/3904757910' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "mgr fail", "who": "smithi053.ygzjut"}]': finished 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: mgrmap e23: smithi110.rivldx(active, starting, since 0.993349s) 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:30:03.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:30:03.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:30:03.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:30:03.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:30:03.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:30:03.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:30:03.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:30:03.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:03 smithi110 ceph-mon[25804]: Manager daemon smithi110.rivldx is now available 2024-09-13T05:30:04.347 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: mgrmap e24: smithi110.rivldx(active, since 2s) 2024-09-13T05:30:04.347 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.347 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: Checking for cert/key for grafana.smithi053 2024-09-13T05:30:04.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:30:04.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:30:04.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:04.349 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:30:04.349 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:30:04.349 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:30:04.349 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/trash_purge_schedule"}]: dispatch 2024-09-13T05:30:04.349 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:04 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/trash_purge_schedule"}]: dispatch 2024-09-13T05:30:04.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 86745 12.98 MB/sec execute 58 sec latency 35.047 ms 2024-09-13T05:30:04.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: mgrmap e24: smithi110.rivldx(active, since 2s) 2024-09-13T05:30:04.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: Checking for cert/key for grafana.smithi053 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/trash_purge_schedule"}]: dispatch 2024-09-13T05:30:04.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:04 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi110.rivldx/trash_purge_schedule"}]: dispatch 2024-09-13T05:30:05.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 87264 12.98 MB/sec execute 59 sec latency 36.303 ms 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: pgmap v4: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: Deploying cephadm binary to smithi110 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: [13/Sep/2024:05:30:04] ENGINE Bus STARTING 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: [13/Sep/2024:05:30:04] ENGINE Serving on http://172.21.15.110:8765 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: [13/Sep/2024:05:30:04] ENGINE Serving on https://172.21.15.110:7150 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: [13/Sep/2024:05:30:04] ENGINE Bus STARTED 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: [13/Sep/2024:05:30:04] ENGINE Error in HTTPServer.serve 2024-09-13T05:30:05.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Traceback (most recent call last): 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._connections.run(self.expiration_interval) 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._run(expiration_interval) 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: s = self.context.wrap_socket( 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T05:30:05.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: return self.sslsocket_class._create( 2024-09-13T05:30:05.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T05:30:05.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self.do_handshake() 2024-09-13T05:30:05.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T05:30:05.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._sslobj.do_handshake() 2024-09-13T05:30:05.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T05:30:05.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:05 smithi053 ceph-mon[23721]: mgrmap e25: smithi110.rivldx(active, since 4s) 2024-09-13T05:30:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: pgmap v4: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T05:30:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: Deploying cephadm binary to smithi110 2024-09-13T05:30:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: [13/Sep/2024:05:30:04] ENGINE Bus STARTING 2024-09-13T05:30:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: [13/Sep/2024:05:30:04] ENGINE Serving on http://172.21.15.110:8765 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: [13/Sep/2024:05:30:04] ENGINE Serving on https://172.21.15.110:7150 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: [13/Sep/2024:05:30:04] ENGINE Bus STARTED 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: [13/Sep/2024:05:30:04] ENGINE Error in HTTPServer.serve 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Traceback (most recent call last): 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._run(expiration_interval) 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T05:30:05.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: s = self.context.wrap_socket( 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: return self.sslsocket_class._create( 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self.do_handshake() 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._sslobj.do_handshake() 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T05:30:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:05 smithi110 ceph-mon[25804]: mgrmap e25: smithi110.rivldx(active, since 4s) 2024-09-13T05:30:06.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 87767 12.91 MB/sec execute 60 sec latency 31.000 ms 2024-09-13T05:30:06.624 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:06 smithi053 ceph-mon[23721]: Deploying cephadm binary to smithi053 2024-09-13T05:30:06.625 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:06 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:06.625 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:06 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:06 smithi110 ceph-mon[25804]: Deploying cephadm binary to smithi053 2024-09-13T05:30:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:06 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:06 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:07.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 88054 12.87 MB/sec execute 61 sec latency 37.765 ms 2024-09-13T05:30:07.640 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:07 smithi053 ceph-mon[23721]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T05:30:07.641 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:07 smithi053 ceph-mon[23721]: mgrmap e26: smithi110.rivldx(active, since 6s) 2024-09-13T05:30:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:07 smithi110 ceph-mon[25804]: pgmap v5: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T05:30:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:07 smithi110 ceph-mon[25804]: mgrmap e26: smithi110.rivldx(active, since 6s) 2024-09-13T05:30:08.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 88514 12.94 MB/sec execute 62 sec latency 40.720 ms 2024-09-13T05:30:09.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 88869 12.93 MB/sec execute 63 sec latency 62.886 ms 2024-09-13T05:30:09.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:09 smithi053 ceph-mon[23721]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T05:30:09.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:09 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:09.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:09 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:09.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:09 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:09.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:09 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:10.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:09 smithi110 ceph-mon[25804]: pgmap v6: 97 pgs: 97 active+clean; 326 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T05:30:10.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:09 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:10.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:09 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:10.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:09 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:10.113 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:09 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:10.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 89212 12.96 MB/sec execute 64 sec latency 41.351 ms 2024-09-13T05:30:11.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 89759 12.81 MB/sec execute 65 sec latency 37.301 ms 2024-09-13T05:30:11.960 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:11 smithi053 ceph-mon[23721]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.6 MiB/s wr, 453 op/s 2024-09-13T05:30:11.960 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:11 smithi053 ceph-mon[23721]: Standby manager daemon smithi053.ygzjut started 2024-09-13T05:30:11.960 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:11 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/crt"}]: dispatch 2024-09-13T05:30:11.960 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:11 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:30:11.960 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:11 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/key"}]: dispatch 2024-09-13T05:30:11.960 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:11 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:30:12.149 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:11 smithi110 ceph-mon[25804]: pgmap v7: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 4.6 MiB/s wr, 453 op/s 2024-09-13T05:30:12.150 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:11 smithi110 ceph-mon[25804]: Standby manager daemon smithi053.ygzjut started 2024-09-13T05:30:12.150 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:11 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/crt"}]: dispatch 2024-09-13T05:30:12.150 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:11 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:30:12.150 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:11 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/key"}]: dispatch 2024-09-13T05:30:12.150 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:11 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/500837210' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:30:12.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 89995 12.69 MB/sec execute 66 sec latency 60.023 ms 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: mgrmap e27: smithi110.rivldx(active, since 10s), standbys: smithi053.ygzjut 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:12.994 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:12 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: mgrmap e27: smithi110.rivldx(active, since 10s), standbys: smithi053.ygzjut 2024-09-13T05:30:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:30:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:13.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:13.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:13.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:13.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:13.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:12 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:13.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 90328 12.66 MB/sec execute 67 sec latency 34.850 ms 2024-09-13T05:30:14.039 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:13 smithi053 ceph-mon[23721]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 351 op/s 2024-09-13T05:30:14.039 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:13 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:14.039 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:13 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:14.039 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:13 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi110 to 2704M 2024-09-13T05:30:14.039 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:13 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:13 smithi110 ceph-mon[25804]: pgmap v8: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 3.5 MiB/s wr, 351 op/s 2024-09-13T05:30:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:13 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:13 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:13 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi110 to 2704M 2024-09-13T05:30:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:13 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:14.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 90730 12.65 MB/sec execute 68 sec latency 56.916 ms 2024-09-13T05:30:15.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 91164 12.57 MB/sec execute 69 sec latency 41.562 ms 2024-09-13T05:30:16.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: pgmap v9: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.5 MiB/s wr, 543 op/s 2024-09-13T05:30:16.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.059 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: Adjusting osd_memory_target on smithi053 to 1937M 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:30:16.060 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:16 smithi053 ceph-mon[23721]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:30:16.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 91449 12.48 MB/sec execute 70 sec latency 64.841 ms 2024-09-13T05:30:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: pgmap v9: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 5.5 MiB/s wr, 543 op/s 2024-09-13T05:30:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: Adjusting osd_memory_target on smithi053 to 1937M 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:30:16.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:16 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:30:17.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 91771 12.51 MB/sec execute 71 sec latency 52.433 ms 2024-09-13T05:30:17.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:30:17.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:30:17.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:30:17.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: pgmap v10: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-13T05:30:17.633 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.634 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Reconfiguring alertmanager.smithi053 (dependencies changed)... 2024-09-13T05:30:17.635 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:17 smithi053 ceph-mon[23721]: Deploying daemon alertmanager.smithi053 on smithi053 2024-09-13T05:30:17.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:30:17.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:30:17.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: pgmap v10: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:17.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Reconfiguring alertmanager.smithi053 (dependencies changed)... 2024-09-13T05:30:17.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:17 smithi110 ceph-mon[25804]: Deploying daemon alertmanager.smithi053 on smithi053 2024-09-13T05:30:18.239 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:30:18.407 INFO:teuthology.orchestra.run.smithi053.stdout: 5 92015 12.46 MB/sec execute 72 sec latency 59.177 ms 2024-09-13T05:30:19.175 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:30:19.175 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (8m) 4s ago 9m 26.0M - 0.23.0 ba2b418f427c 2a3ae76369c5 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (9m) 4s ago 9m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (9m) 10s ago 9m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (8m) 4s ago 9m 49.3M - dad864ee21e9 41e920907455 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (3m) 4s ago 3m 36.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (3m) 10s ago 3m 14.6M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (11m) 4s ago 11m 227M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (37s) 10s ago 9m 496M - 19.3.0-4911-g9d608eb1 0ce38589d192 e879e7d7efd2 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (11m) 4s ago 11m 48.8M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (9m) 10s ago 9m 40.5M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (3m) 4s ago 3m 254M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (3m) 10s ago 3m 83.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (9m) 4s ago 9m 19.5M - 1.3.1 1dbe0e931976 aa1768a447f0 2024-09-13T05:30:19.176 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (8m) 10s ago 8m 18.1M - 1.3.1 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (8m) 4s ago 8m 373M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (7m) 4s ago 7m 272M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (7m) 4s ago 7m 221M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (6m) 4s ago 6m 331M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (6m) 10s ago 6m 331M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (6m) 10s ago 6m 343M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (5m) 10s ago 5m 291M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (5m) 10s ago 5m 435M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:30:19.177 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (8m) 4s ago 8m 64.4M - 2.33.4 514e6a882f6e 7c997d72a66c 2024-09-13T05:30:19.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 92309 12.42 MB/sec execute 73 sec latency 62.030 ms 2024-09-13T05:30:19.425 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:19 smithi053 ceph-mon[23721]: pgmap v11: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-13T05:30:19.425 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:19 smithi053 ceph-mon[23721]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:19.426 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:19 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:19.426 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:19 smithi053 ceph-mon[23721]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:19 smithi110 ceph-mon[25804]: pgmap v11: 97 pgs: 97 active+clean; 318 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-13T05:30:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:19 smithi110 ceph-mon[25804]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:19 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:19 smithi110 ceph-mon[25804]: from='client.14776 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:19.814 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:30:19.815 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:30:19.816 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "", 2024-09-13T05:30:20.372 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:30:20.373 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:30:20.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 92787 12.51 MB/sec execute 74 sec latency 43.101 ms 2024-09-13T05:30:20.698 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:20 smithi053 ceph-mon[23721]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:20.698 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:20 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/744358845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:30:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:20 smithi110 ceph-mon[25804]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:20 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/744358845' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:30:21.009 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:30:21.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 93182 12.37 MB/sec execute 75 sec latency 67.389 ms 2024-09-13T05:30:21.735 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:21 smithi053 ceph-mon[23721]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.0 MiB/s wr, 683 op/s 2024-09-13T05:30:21.735 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:21 smithi053 ceph-mon[23721]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:21.735 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:21 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/2525209540' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:30:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:21 smithi110 ceph-mon[25804]: pgmap v12: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 7.0 MiB/s wr, 683 op/s 2024-09-13T05:30:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:21 smithi110 ceph-mon[25804]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:21 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/2525209540' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:30:22.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 93529 12.27 MB/sec execute 76 sec latency 67.983 ms 2024-09-13T05:30:23.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 93801 12.22 MB/sec execute 77 sec latency 85.660 ms 2024-09-13T05:30:23.757 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:23 smithi053 ceph-mon[23721]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 422 op/s 2024-09-13T05:30:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:23 smithi110 ceph-mon[25804]: pgmap v13: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 422 op/s 2024-09-13T05:30:24.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 94290 12.27 MB/sec execute 78 sec latency 33.560 ms 2024-09-13T05:30:25.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 94737 12.20 MB/sec execute 79 sec latency 37.632 ms 2024-09-13T05:30:25.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:25 smithi053 ceph-mon[23721]: pgmap v14: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 655 op/s 2024-09-13T05:30:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:25 smithi110 ceph-mon[25804]: pgmap v14: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 655 op/s 2024-09-13T05:30:26.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 95074 12.18 MB/sec execute 80 sec latency 54.516 ms 2024-09-13T05:30:27.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 95447 12.21 MB/sec execute 81 sec latency 47.828 ms 2024-09-13T05:30:27.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:27 smithi053 ceph-mon[23721]: pgmap v15: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-13T05:30:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:27 smithi110 ceph-mon[25804]: pgmap v15: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-13T05:30:28.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 95764 12.18 MB/sec execute 82 sec latency 54.348 ms 2024-09-13T05:30:29.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 96012 12.13 MB/sec execute 83 sec latency 53.233 ms 2024-09-13T05:30:29.896 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:29 smithi053 ceph-mon[23721]: pgmap v16: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-13T05:30:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:29 smithi110 ceph-mon[25804]: pgmap v16: 97 pgs: 97 active+clean; 319 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-13T05:30:30.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 96279 12.10 MB/sec execute 84 sec latency 75.554 ms 2024-09-13T05:30:31.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 96434 12.02 MB/sec execute 85 sec latency 87.547 ms 2024-09-13T05:30:31.700 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:31 smithi053 ceph-mon[23721]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 638 op/s 2024-09-13T05:30:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:31 smithi110 ceph-mon[25804]: pgmap v17: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.6 MiB/s wr, 638 op/s 2024-09-13T05:30:32.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 96626 11.90 MB/sec execute 86 sec latency 64.289 ms 2024-09-13T05:30:33.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 96968 11.80 MB/sec execute 87 sec latency 48.236 ms 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 448 op/s 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: Reconfiguring grafana.smithi053 (dependencies changed)... 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:30:33.982 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:33 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: pgmap v18: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 448 op/s 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: Reconfiguring grafana.smithi053 (dependencies changed)... 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:30:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:33 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:34.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 97213 11.73 MB/sec execute 88 sec latency 69.894 ms 2024-09-13T05:30:34.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:34 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:30:34.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:34 smithi053 ceph-mon[23721]: Reconfiguring daemon grafana.smithi053 on smithi053 2024-09-13T05:30:34.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:34 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T05:30:34.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:34 smithi110 ceph-mon[25804]: Reconfiguring daemon grafana.smithi053 on smithi053 2024-09-13T05:30:35.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 97530 11.70 MB/sec execute 89 sec latency 66.666 ms 2024-09-13T05:30:35.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:35 smithi053 ceph-mon[23721]: pgmap v19: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-09-13T05:30:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:35 smithi110 ceph-mon[25804]: pgmap v19: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-09-13T05:30:36.408 INFO:teuthology.orchestra.run.smithi053.stdout: 5 97838 11.70 MB/sec execute 90 sec latency 58.713 ms 2024-09-13T05:30:37.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 98088 11.60 MB/sec execute 91 sec latency 74.739 ms 2024-09-13T05:30:37.841 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:37 smithi053 ceph-mon[23721]: pgmap v20: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 367 op/s 2024-09-13T05:30:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:37 smithi110 ceph-mon[25804]: pgmap v20: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 367 op/s 2024-09-13T05:30:38.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 98497 11.58 MB/sec execute 92 sec latency 48.547 ms 2024-09-13T05:30:39.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 98677 11.51 MB/sec execute 93 sec latency 66.309 ms 2024-09-13T05:30:39.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:39 smithi053 ceph-mon[23721]: pgmap v21: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 367 op/s 2024-09-13T05:30:40.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:39 smithi110 ceph-mon[25804]: pgmap v21: 97 pgs: 97 active+clean; 325 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 367 op/s 2024-09-13T05:30:40.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 98887 11.50 MB/sec execute 94 sec latency 61.122 ms 2024-09-13T05:30:41.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 99139 11.47 MB/sec execute 95 sec latency 75.290 ms 2024-09-13T05:30:41.979 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:41 smithi053 ceph-mon[23721]: pgmap v22: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 548 op/s 2024-09-13T05:30:41.979 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:41 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:41.979 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:41 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:41 smithi110 ceph-mon[25804]: pgmap v22: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 548 op/s 2024-09-13T05:30:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:41 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:41 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:42.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 99414 11.43 MB/sec execute 96 sec latency 54.780 ms 2024-09-13T05:30:42.996 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:42 smithi053 ceph-mon[23721]: Reconfiguring node-exporter.smithi053 (dependencies changed)... 2024-09-13T05:30:42.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:42 smithi053 ceph-mon[23721]: Deploying daemon node-exporter.smithi053 on smithi053 2024-09-13T05:30:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:42 smithi110 ceph-mon[25804]: Reconfiguring node-exporter.smithi053 (dependencies changed)... 2024-09-13T05:30:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:42 smithi110 ceph-mon[25804]: Deploying daemon node-exporter.smithi053 on smithi053 2024-09-13T05:30:43.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 99701 11.44 MB/sec execute 97 sec latency 62.469 ms 2024-09-13T05:30:44.016 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:43 smithi053 ceph-mon[23721]: pgmap v23: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.9 MiB/s wr, 333 op/s 2024-09-13T05:30:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:43 smithi110 ceph-mon[25804]: pgmap v23: 97 pgs: 97 active+clean; 320 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 2.9 MiB/s wr, 333 op/s 2024-09-13T05:30:44.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 99916 11.37 MB/sec execute 98 sec latency 48.110 ms 2024-09-13T05:30:45.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 100193 11.31 MB/sec execute 99 sec latency 48.563 ms 2024-09-13T05:30:45.843 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:45 smithi053 ceph-mon[23721]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-09-13T05:30:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:45 smithi110 ceph-mon[25804]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.9 MiB/s wr, 484 op/s 2024-09-13T05:30:46.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 100567 11.26 MB/sec execute 100 sec latency 69.835 ms 2024-09-13T05:30:47.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 100859 11.18 MB/sec execute 101 sec latency 92.639 ms 2024-09-13T05:30:47.869 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:47 smithi053 ceph-mon[23721]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 331 op/s 2024-09-13T05:30:48.045 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:47 smithi110 ceph-mon[25804]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 331 op/s 2024-09-13T05:30:48.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 101203 11.19 MB/sec execute 102 sec latency 39.926 ms 2024-09-13T05:30:49.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 101578 11.17 MB/sec execute 103 sec latency 60.433 ms 2024-09-13T05:30:49.413 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:49 smithi053 ceph-mon[23721]: pgmap v26: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 331 op/s 2024-09-13T05:30:49.413 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:49 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:49.413 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:49 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:49.413 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:49 smithi053 ceph-mon[23721]: Reconfiguring prometheus.smithi053 (dependencies changed)... 2024-09-13T05:30:49.413 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:49 smithi053 ceph-mon[23721]: Deploying daemon prometheus.smithi053 on smithi053 2024-09-13T05:30:49.413 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:49 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:49.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:49 smithi110 ceph-mon[25804]: pgmap v26: 97 pgs: 97 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 331 op/s 2024-09-13T05:30:49.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:49 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:49.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:49 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:30:49.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:49 smithi110 ceph-mon[25804]: Reconfiguring prometheus.smithi053 (dependencies changed)... 2024-09-13T05:30:49.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:49 smithi110 ceph-mon[25804]: Deploying daemon prometheus.smithi053 on smithi053 2024-09-13T05:30:49.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:49 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:30:50.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 102080 11.18 MB/sec execute 104 sec latency 45.412 ms 2024-09-13T05:30:51.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 102285 11.13 MB/sec execute 105 sec latency 42.943 ms 2024-09-13T05:30:51.544 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:30:51.697 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:51 smithi053 ceph-mon[23721]: pgmap v27: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-09-13T05:30:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:51 smithi110 ceph-mon[25804]: pgmap v27: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.2 MiB/s wr, 526 op/s 2024-09-13T05:30:52.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 102643 11.18 MB/sec execute 106 sec latency 40.183 ms 2024-09-13T05:30:52.552 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 starting - - - - 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (10m) 37s ago 10m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (9m) 43s ago 9m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 starting - - - - 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (4m) 37s ago 4m 36.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (4m) 43s ago 4m 14.6M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:9283 running (11m) 37s ago 11m 227M - 17.2.0 e1d6a67b021e f027c28bd2b1 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (70s) 43s ago 9m 496M - 19.3.0-4911-g9d608eb1 0ce38589d192 e879e7d7efd2 2024-09-13T05:30:52.553 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (11m) 37s ago 11m 48.8M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (9m) 43s ago 9m 40.5M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (3m) 37s ago 3m 254M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (3m) 43s ago 3m 83.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 starting - - - - 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (9m) 43s ago 9m 18.1M - 1.3.1 1dbe0e931976 0f34fdb31cc8 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (8m) 37s ago 8m 373M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (8m) 37s ago 8m 272M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (7m) 37s ago 7m 221M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (7m) 37s ago 7m 331M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (7m) 43s ago 7m 331M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (6m) 43s ago 6m 343M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:30:52.554 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (6m) 43s ago 6m 291M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:30:52.555 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (5m) 43s ago 5m 435M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:30:52.555 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (9m) 37s ago 9m 64.4M - 2.33.4 514e6a882f6e 7c997d72a66c 2024-09-13T05:30:52.730 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:52 smithi053 ceph-mon[23721]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:52.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:52 smithi110 ceph-mon[25804]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:30:53.179 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:30:53.180 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:30:53.180 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:30:53.180 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T05:30:53.180 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:30:53.180 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:30:53.180 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:30:53.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 102927 11.14 MB/sec execute 107 sec latency 45.271 ms 2024-09-13T05:30:53.659 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:30:53.659 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:30:53.659 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:30:53.659 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:30:53.659 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:30:53.660 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T05:30:53.660 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "", 2024-09-13T05:30:53.660 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:30:53.660 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:30:53.776 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:53 smithi053 ceph-mon[23721]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:53.776 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:53 smithi053 ceph-mon[23721]: pgmap v28: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 345 op/s 2024-09-13T05:30:53.776 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:53 smithi053 ceph-mon[23721]: from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:53.776 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:53 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3202191263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:30:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:53 smithi110 ceph-mon[25804]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:53 smithi110 ceph-mon[25804]: pgmap v28: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.8 MiB/s wr, 345 op/s 2024-09-13T05:30:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:53 smithi110 ceph-mon[25804]: from='client.14804 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:53 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3202191263' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:30:54.407 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:30:54.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 103138 11.12 MB/sec execute 108 sec latency 50.037 ms 2024-09-13T05:30:54.789 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:54 smithi053 ceph-mon[23721]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:54.789 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:54 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/83872224' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:30:54.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:54 smithi110 ceph-mon[25804]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:30:54.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:54 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/83872224' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:30:55.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 103396 11.09 MB/sec execute 109 sec latency 68.330 ms 2024-09-13T05:30:55.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:55 smithi053 ceph-mon[23721]: pgmap v29: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 538 op/s 2024-09-13T05:30:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:55 smithi110 ceph-mon[25804]: pgmap v29: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.9 MiB/s wr, 538 op/s 2024-09-13T05:30:56.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 103694 11.05 MB/sec execute 110 sec latency 34.039 ms 2024-09-13T05:30:57.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 104123 11.00 MB/sec execute 111 sec latency 80.664 ms 2024-09-13T05:30:57.834 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:57 smithi053 ceph-mon[23721]: pgmap v30: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 386 op/s 2024-09-13T05:30:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:57 smithi110 ceph-mon[25804]: pgmap v30: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 386 op/s 2024-09-13T05:30:58.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 104550 10.97 MB/sec execute 112 sec latency 40.223 ms 2024-09-13T05:30:59.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 104983 11.01 MB/sec execute 113 sec latency 38.772 ms 2024-09-13T05:30:59.883 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:30:59 smithi053 ceph-mon[23721]: pgmap v31: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 386 op/s 2024-09-13T05:30:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:30:59 smithi110 ceph-mon[25804]: pgmap v31: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 386 op/s 2024-09-13T05:31:00.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 105675 11.05 MB/sec execute 114 sec latency 66.495 ms 2024-09-13T05:31:01.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 105973 11.04 MB/sec execute 115 sec latency 41.263 ms 2024-09-13T05:31:01.667 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:01 smithi053 ceph-mon[23721]: pgmap v32: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 595 op/s 2024-09-13T05:31:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:01 smithi110 ceph-mon[25804]: pgmap v32: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 595 op/s 2024-09-13T05:31:02.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 106493 11.10 MB/sec execute 116 sec latency 32.759 ms 2024-09-13T05:31:03.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 106934 11.14 MB/sec execute 117 sec latency 32.500 ms 2024-09-13T05:31:03.695 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:03 smithi053 ceph-mon[23721]: pgmap v33: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 401 op/s 2024-09-13T05:31:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:03 smithi110 ceph-mon[25804]: pgmap v33: 97 pgs: 97 active+clean; 326 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 401 op/s 2024-09-13T05:31:04.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 107208 11.11 MB/sec execute 118 sec latency 63.908 ms 2024-09-13T05:31:04.701 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:04 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:04 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:05.409 INFO:teuthology.orchestra.run.smithi053.stdout: 5 107602 11.06 MB/sec execute 119 sec latency 44.360 ms 2024-09-13T05:31:05.731 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:05 smithi053 ceph-mon[23721]: pgmap v34: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 669 op/s 2024-09-13T05:31:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:05 smithi110 ceph-mon[25804]: pgmap v34: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.1 MiB/s wr, 669 op/s 2024-09-13T05:31:06.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 107853 10.99 MB/sec execute 120 sec latency 51.174 ms 2024-09-13T05:31:07.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 108132 10.97 MB/sec execute 121 sec latency 83.740 ms 2024-09-13T05:31:07.757 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:07 smithi053 ceph-mon[23721]: pgmap v35: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 476 op/s 2024-09-13T05:31:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:07 smithi110 ceph-mon[25804]: pgmap v35: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 476 op/s 2024-09-13T05:31:08.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 108536 11.00 MB/sec execute 122 sec latency 40.250 ms 2024-09-13T05:31:09.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 108900 10.97 MB/sec execute 123 sec latency 93.048 ms 2024-09-13T05:31:09.818 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:09 smithi053 ceph-mon[23721]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 476 op/s 2024-09-13T05:31:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:09 smithi110 ceph-mon[25804]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 476 op/s 2024-09-13T05:31:10.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 109222 10.93 MB/sec execute 124 sec latency 56.593 ms 2024-09-13T05:31:11.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 109447 10.91 MB/sec execute 125 sec latency 62.770 ms 2024-09-13T05:31:12.086 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:11 smithi110 ceph-mon[25804]: pgmap v37: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-09-13T05:31:12.086 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:11 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:12.086 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:11 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:12.086 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:11 smithi110 ceph-mon[25804]: Reconfiguring node-exporter.smithi110 (dependencies changed)... 2024-09-13T05:31:12.086 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:11 smithi110 ceph-mon[25804]: Deploying daemon node-exporter.smithi110 on smithi110 2024-09-13T05:31:12.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:11 smithi053 ceph-mon[23721]: pgmap v37: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 685 op/s 2024-09-13T05:31:12.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:11 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:12.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:11 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:12.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:11 smithi053 ceph-mon[23721]: Reconfiguring node-exporter.smithi110 (dependencies changed)... 2024-09-13T05:31:12.121 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:11 smithi053 ceph-mon[23721]: Deploying daemon node-exporter.smithi110 on smithi110 2024-09-13T05:31:12.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 109807 10.94 MB/sec execute 126 sec latency 39.959 ms 2024-09-13T05:31:13.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 110117 10.91 MB/sec execute 127 sec latency 87.380 ms 2024-09-13T05:31:13.945 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:13 smithi053 ceph-mon[23721]: pgmap v38: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-09-13T05:31:14.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:13 smithi110 ceph-mon[25804]: pgmap v38: 97 pgs: 97 active+clean; 325 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 476 op/s 2024-09-13T05:31:14.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 110454 10.92 MB/sec execute 128 sec latency 50.193 ms 2024-09-13T05:31:15.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 110688 10.89 MB/sec execute 129 sec latency 77.969 ms 2024-09-13T05:31:16.009 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:15 smithi053 ceph-mon[23721]: pgmap v39: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 659 op/s 2024-09-13T05:31:16.009 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:15 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:16.009 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:15 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:16.009 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:15 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:31:16.009 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:15 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi053.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T05:31:16.009 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:15 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:16.009 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:15 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:31:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:15 smithi110 ceph-mon[25804]: pgmap v39: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.9 MiB/s wr, 659 op/s 2024-09-13T05:31:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:15 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:15 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:15 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:31:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:15 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi053.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T05:31:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:15 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:15 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:31:16.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 110996 10.83 MB/sec execute 130 sec latency 41.075 ms 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi053.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T05:31:17.027 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:17.028 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:17.028 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:17.028 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:16 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi053.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi053.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T05:31:17.227 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi053.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T05:31:17.228 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:17.228 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:17.228 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:17.228 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:16 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:17.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 111285 10.79 MB/sec execute 131 sec latency 57.470 ms 2024-09-13T05:31:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:17 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:17 smithi053 ceph-mon[23721]: pgmap v40: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 391 op/s 2024-09-13T05:31:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:17 smithi053 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi110.rivldx) 2024-09-13T05:31:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:17 smithi053 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi110.rivldx) 2024-09-13T05:31:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:17 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:17.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:17 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:17 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:17 smithi110 ceph-mon[25804]: pgmap v40: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 391 op/s 2024-09-13T05:31:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:17 smithi110 ceph-mon[25804]: Upgrade: Need to upgrade myself (mgr.smithi110.rivldx) 2024-09-13T05:31:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:17 smithi110 ceph-mon[25804]: Upgrade: Need to upgrade myself (mgr.smithi110.rivldx) 2024-09-13T05:31:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:17 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:18.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:17 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:18.410 INFO:teuthology.orchestra.run.smithi053.stdout: 5 111634 10.77 MB/sec execute 132 sec latency 65.500 ms 2024-09-13T05:31:19.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:18 smithi053 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:19.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:18 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:19.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:18 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:31:19.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:18 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:31:19.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:18 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:31:19.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:18 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:31:19.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:18 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:19.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:18 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:19.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:18 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:19.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:18 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:31:19.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:18 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:31:19.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:18 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:31:19.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:18 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:31:19.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:18 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:19.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 112015 10.78 MB/sec execute 133 sec latency 47.187 ms 2024-09-13T05:31:20.015 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:19 smithi053 ceph-mon[23721]: pgmap v41: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 391 op/s 2024-09-13T05:31:20.015 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:19 smithi053 ceph-mon[23721]: Upgrade: Updating mgr.smithi053.ygzjut 2024-09-13T05:31:20.015 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:19 smithi053 ceph-mon[23721]: Deploying daemon mgr.smithi053.ygzjut on smithi053 2024-09-13T05:31:20.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:19 smithi110 ceph-mon[25804]: pgmap v41: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 391 op/s 2024-09-13T05:31:20.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:19 smithi110 ceph-mon[25804]: Upgrade: Updating mgr.smithi053.ygzjut 2024-09-13T05:31:20.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:19 smithi110 ceph-mon[25804]: Deploying daemon mgr.smithi053.ygzjut on smithi053 2024-09-13T05:31:20.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 112528 10.80 MB/sec execute 134 sec latency 53.422 ms 2024-09-13T05:31:21.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 112979 10.80 MB/sec execute 135 sec latency 53.223 ms 2024-09-13T05:31:22.035 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:21 smithi053 ceph-mon[23721]: pgmap v42: 97 pgs: 97 active+clean; 326 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.6 MiB/s wr, 581 op/s 2024-09-13T05:31:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:21 smithi110 ceph-mon[25804]: pgmap v42: 97 pgs: 97 active+clean; 326 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.6 MiB/s wr, 581 op/s 2024-09-13T05:31:22.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 113376 10.84 MB/sec execute 136 sec latency 43.830 ms 2024-09-13T05:31:23.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 113871 10.87 MB/sec execute 137 sec latency 54.814 ms 2024-09-13T05:31:24.066 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:23 smithi053 ceph-mon[23721]: pgmap v43: 97 pgs: 97 active+clean; 326 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 373 op/s 2024-09-13T05:31:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:23 smithi110 ceph-mon[25804]: pgmap v43: 97 pgs: 97 active+clean; 326 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.7 MiB/s wr, 373 op/s 2024-09-13T05:31:24.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 114284 10.88 MB/sec execute 138 sec latency 33.078 ms 2024-09-13T05:31:25.067 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:31:25.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 114552 10.84 MB/sec execute 139 sec latency 116.726 ms 2024-09-13T05:31:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:25 smithi110 ceph-mon[25804]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.7 MiB/s wr, 629 op/s 2024-09-13T05:31:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:25 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:25 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:25 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:25 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:25.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:25 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:26.089 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:25 smithi053 ceph-mon[23721]: pgmap v44: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.7 MiB/s wr, 629 op/s 2024-09-13T05:31:26.089 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:25 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:26.089 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:25 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:26.089 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:25 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:26.089 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:25 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:26.089 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:25 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:26.314 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 starting - - - - 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (10m) 71s ago 10m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (10m) 77s ago 10m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 starting - - - - 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (4m) 71s ago 4m 36.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (4m) 77s ago 4m 14.6M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 starting - - - - 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (104s) 77s ago 10m 496M - 19.3.0-4911-g9d608eb1 0ce38589d192 e879e7d7efd2 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (12m) 71s ago 12m 48.8M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (10m) 77s ago 10m 40.5M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (4m) 71s ago 4m 254M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:31:26.315 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (4m) 77s ago 4m 83.6M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 starting - - - - 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 starting - - - - 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (9m) 71s ago 9m 373M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (8m) 71s ago 8m 272M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (8m) 71s ago 8m 221M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (7m) 71s ago 7m 331M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (7m) 77s ago 7m 331M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (7m) 77s ago 7m 343M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (6m) 77s ago 6m 291M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (6m) 77s ago 6m 435M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:31:26.316 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 starting - - - - 2024-09-13T05:31:26.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 114914 10.78 MB/sec execute 140 sec latency 44.884 ms 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:31:27.030 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:31:27.031 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:31:27.095 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:26 smithi053 ceph-mon[23721]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:27.096 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:26 smithi053 ceph-mon[23721]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:26 smithi110 ceph-mon[25804]: from='client.14820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:26 smithi110 ceph-mon[25804]: from='client.24553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:27.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 115259 10.78 MB/sec execute 141 sec latency 57.721 ms 2024-09-13T05:31:27.616 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:31:27.616 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:31:27.616 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:31:27.616 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:31:27.617 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:31:27.617 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "", 2024-09-13T05:31:27.617 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading mgr daemons", 2024-09-13T05:31:27.617 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:31:27.617 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:31:27.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:27 smithi053 ceph-mon[23721]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-09-13T05:31:27.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:27 smithi053 ceph-mon[23721]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:27.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:27 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/3406632086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:31:28.239 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:27 smithi110 ceph-mon[25804]: pgmap v45: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-09-13T05:31:28.239 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:27 smithi110 ceph-mon[25804]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:28.239 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:27 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/3406632086' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:31:28.338 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:31:28.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 115683 10.82 MB/sec execute 142 sec latency 36.543 ms 2024-09-13T05:31:29.129 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:28 smithi053 ceph-mon[23721]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:29.129 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:28 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/37175367' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:31:29.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:28 smithi110 ceph-mon[25804]: from='client.14836 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:31:29.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:28 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/37175367' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:31:29.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 116100 10.79 MB/sec execute 143 sec latency 67.261 ms 2024-09-13T05:31:30.135 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:29 smithi053 ceph-mon[23721]: pgmap v46: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-09-13T05:31:30.135 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:29 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:30.135 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:29 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:29 smithi110 ceph-mon[25804]: pgmap v46: 97 pgs: 97 active+clean; 327 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-09-13T05:31:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:29 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:30.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:29 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:30.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 116753 10.84 MB/sec execute 144 sec latency 70.339 ms 2024-09-13T05:31:31.411 INFO:teuthology.orchestra.run.smithi053.stdout: 5 117020 10.84 MB/sec execute 145 sec latency 68.338 ms 2024-09-13T05:31:31.930 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:31 smithi053 ceph-mon[23721]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 676 op/s 2024-09-13T05:31:32.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:31 smithi110 ceph-mon[25804]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 676 op/s 2024-09-13T05:31:32.415 INFO:teuthology.orchestra.run.smithi053.stdout: 5 117398 10.84 MB/sec execute 146 sec latency 42.990 ms 2024-09-13T05:31:32.936 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:32 smithi053 ceph-mon[23721]: Standby manager daemon smithi053.ygzjut restarted 2024-09-13T05:31:32.937 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:32 smithi053 ceph-mon[23721]: Standby manager daemon smithi053.ygzjut started 2024-09-13T05:31:32.937 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:32 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/crt"}]: dispatch 2024-09-13T05:31:32.937 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:32 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:31:32.937 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:32 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/key"}]: dispatch 2024-09-13T05:31:32.937 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:32 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:31:33.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:32 smithi110 ceph-mon[25804]: Standby manager daemon smithi053.ygzjut restarted 2024-09-13T05:31:33.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:32 smithi110 ceph-mon[25804]: Standby manager daemon smithi053.ygzjut started 2024-09-13T05:31:33.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:32 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/crt"}]: dispatch 2024-09-13T05:31:33.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:32 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:31:33.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:32 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi053.ygzjut/key"}]: dispatch 2024-09-13T05:31:33.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:32 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:31:33.415 INFO:teuthology.orchestra.run.smithi053.stdout: 5 118004 10.89 MB/sec execute 147 sec latency 27.385 ms 2024-09-13T05:31:33.952 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:33 smithi053 ceph-mon[23721]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 485 op/s 2024-09-13T05:31:33.952 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:33 smithi053 ceph-mon[23721]: mgrmap e28: smithi110.rivldx(active, since 91s), standbys: smithi053.ygzjut 2024-09-13T05:31:33.952 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:33 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:33.952 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:33 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:33.952 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:33 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:33 smithi110 ceph-mon[25804]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 485 op/s 2024-09-13T05:31:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:33 smithi110 ceph-mon[25804]: mgrmap e28: smithi110.rivldx(active, since 91s), standbys: smithi053.ygzjut 2024-09-13T05:31:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:33 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:33 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:34.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:33 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:34.415 INFO:teuthology.orchestra.run.smithi053.stdout: 5 118430 10.84 MB/sec execute 148 sec latency 39.322 ms 2024-09-13T05:31:35.415 INFO:teuthology.orchestra.run.smithi053.stdout: 5 118792 10.83 MB/sec execute 149 sec latency 40.016 ms 2024-09-13T05:31:35.477 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:35 smithi053 ceph-mon[23721]: pgmap v49: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 749 op/s 2024-09-13T05:31:35.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:35 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:35.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:35 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:35.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:35 smithi110 ceph-mon[25804]: pgmap v49: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 8.0 MiB/s wr, 749 op/s 2024-09-13T05:31:35.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:35 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:35.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:35 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:36.415 INFO:teuthology.orchestra.run.smithi053.stdout: 5 119411 10.91 MB/sec execute 150 sec latency 21.954 ms 2024-09-13T05:31:36.489 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:36 smithi053 ceph-mon[23721]: mgrmap e29: smithi110.rivldx(active, since 94s), standbys: smithi053.ygzjut 2024-09-13T05:31:36.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:36 smithi110 ceph-mon[25804]: mgrmap e29: smithi110.rivldx(active, since 94s), standbys: smithi053.ygzjut 2024-09-13T05:31:37.415 INFO:teuthology.orchestra.run.smithi053.stdout: 5 120134 10.93 MB/sec execute 151 sec latency 19.314 ms 2024-09-13T05:31:37.494 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:37 smithi053 ceph-mon[23721]: pgmap v50: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-13T05:31:37.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:37 smithi110 ceph-mon[25804]: pgmap v50: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 493 op/s 2024-09-13T05:31:38.416 INFO:teuthology.orchestra.run.smithi053.stdout: 5 120620 10.97 MB/sec execute 152 sec latency 40.148 ms 2024-09-13T05:31:39.416 INFO:teuthology.orchestra.run.smithi053.stdout: 5 121006 10.97 MB/sec execute 153 sec latency 53.151 ms 2024-09-13T05:31:39.521 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:39 smithi053 ceph-mon[23721]: pgmap v51: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 630 op/s 2024-09-13T05:31:39.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:39 smithi110 ceph-mon[25804]: pgmap v51: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 630 op/s 2024-09-13T05:31:40.416 INFO:teuthology.orchestra.run.smithi053.stdout: 5 121385 10.99 MB/sec execute 154 sec latency 88.310 ms 2024-09-13T05:31:41.416 INFO:teuthology.orchestra.run.smithi053.stdout: 5 121685 10.96 MB/sec execute 155 sec latency 52.455 ms 2024-09-13T05:31:42.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 776 op/s 2024-09-13T05:31:42.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:31:42.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:31:42.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:42.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr fail", "who": "smithi110.rivldx"}]: dispatch 2024-09-13T05:31:42.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr fail", "who": "smithi110.rivldx"}]: dispatch 2024-09-13T05:31:42.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:41 smithi053 ceph-mon[23721]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T05:31:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 776 op/s 2024-09-13T05:31:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:31:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:31:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' 2024-09-13T05:31:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 172.21.15.110:0/2781310966' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr fail", "who": "smithi110.rivldx"}]: dispatch 2024-09-13T05:31:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "mgr fail", "who": "smithi110.rivldx"}]: dispatch 2024-09-13T05:31:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:41 smithi110 ceph-mon[25804]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T05:31:42.416 INFO:teuthology.orchestra.run.smithi053.stdout: 5 121982 10.90 MB/sec execute 156 sec latency 68.435 ms 2024-09-13T05:31:43.104 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd='[{"prefix": "mgr fail", "who": "smithi110.rivldx"}]': finished 2024-09-13T05:31:43.104 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: mgrmap e30: smithi053.ygzjut(active, starting, since 0.683519s) 2024-09-13T05:31:43.104 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:31:43.105 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:31:43.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:31:43.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:31:43.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:31:43.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:42 smithi053 ceph-mon[23721]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:31:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.24481 ' entity='mgr.smithi110.rivldx' cmd='[{"prefix": "mgr fail", "who": "smithi110.rivldx"}]': finished 2024-09-13T05:31:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: mgrmap e30: smithi053.ygzjut(active, starting, since 0.683519s) 2024-09-13T05:31:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:31:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T05:31:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T05:31:43.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T05:31:43.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T05:31:43.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:42 smithi110 ceph-mon[25804]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:31:43.419 INFO:teuthology.orchestra.run.smithi053.stdout: 5 122278 10.89 MB/sec execute 157 sec latency 51.386 ms 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: mgrmap e31: smithi053.ygzjut(active, since 1.69124s) 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:31:44.142 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:43 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:31:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: mgrmap e31: smithi053.ygzjut(active, since 1.69124s) 2024-09-13T05:31:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: pgmap v3: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:31:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:44.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:44.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:31:44.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"}]: dispatch 2024-09-13T05:31:44.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:43 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"}]: dispatch 2024-09-13T05:31:44.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 122549 10.87 MB/sec execute 158 sec latency 58.473 ms 2024-09-13T05:31:44.977 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:44 smithi053 ceph-mon[23721]: pgmap v4: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:44 smithi110 ceph-mon[25804]: pgmap v4: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:45.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 122884 10.88 MB/sec execute 159 sec latency 66.584 ms 2024-09-13T05:31:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:46 smithi110 ceph-mon[25804]: [13/Sep/2024:05:31:44] ENGINE Bus STARTING 2024-09-13T05:31:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:46 smithi110 ceph-mon[25804]: mgrmap e32: smithi053.ygzjut(active, since 3s) 2024-09-13T05:31:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:46 smithi110 ceph-mon[25804]: [13/Sep/2024:05:31:44] ENGINE Serving on http://172.21.15.53:8765 2024-09-13T05:31:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:46 smithi110 ceph-mon[25804]: [13/Sep/2024:05:31:45] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:31:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:46 smithi110 ceph-mon[25804]: [13/Sep/2024:05:31:45] ENGINE Bus STARTED 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:46 smithi110 ceph-mon[25804]: [13/Sep/2024:05:31:45] ENGINE Error in HTTPServer.serve 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Traceback (most recent call last): 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._run(expiration_interval) 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T05:31:46.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: s = self.context.wrap_socket( 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: return self.sslsocket_class._create( 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self.do_handshake() 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._sslobj.do_handshake() 2024-09-13T05:31:46.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T05:31:46.249 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:46 smithi053 ceph-mon[23721]: [13/Sep/2024:05:31:44] ENGINE Bus STARTING 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:46 smithi053 ceph-mon[23721]: mgrmap e32: smithi053.ygzjut(active, since 3s) 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:46 smithi053 ceph-mon[23721]: [13/Sep/2024:05:31:44] ENGINE Serving on http://172.21.15.53:8765 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:46 smithi053 ceph-mon[23721]: [13/Sep/2024:05:31:45] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:46 smithi053 ceph-mon[23721]: [13/Sep/2024:05:31:45] ENGINE Bus STARTED 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:46 smithi053 ceph-mon[23721]: [13/Sep/2024:05:31:45] ENGINE Error in HTTPServer.serve 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Traceback (most recent call last): 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._connections.run(self.expiration_interval) 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._run(expiration_interval) 2024-09-13T05:31:46.250 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: s = self.context.wrap_socket( 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: return self.sslsocket_class._create( 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self.do_handshake() 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._sslobj.do_handshake() 2024-09-13T05:31:46.251 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T05:31:46.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 123223 10.85 MB/sec execute 160 sec latency 70.048 ms 2024-09-13T05:31:47.263 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:47 smithi053 ceph-mon[23721]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:47.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 123562 10.83 MB/sec execute 161 sec latency 53.323 ms 2024-09-13T05:31:47.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:47 smithi110 ceph-mon[25804]: pgmap v5: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:48.275 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:48 smithi053 ceph-mon[23721]: mgrmap e33: smithi053.ygzjut(active, since 5s) 2024-09-13T05:31:48.275 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:48 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:48.276 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:48 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:48.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 123823 10.81 MB/sec execute 162 sec latency 43.350 ms 2024-09-13T05:31:48.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:48 smithi110 ceph-mon[25804]: mgrmap e33: smithi053.ygzjut(active, since 5s) 2024-09-13T05:31:48.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:48 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:48.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:48 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:49.287 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:49 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:49.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 124083 10.79 MB/sec execute 163 sec latency 60.168 ms 2024-09-13T05:31:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: pgmap v6: 97 pgs: 97 active+clean; 333 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:31:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:31:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:31:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:31:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:31:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2550924681' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:31:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:49.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:49 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:50.342 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:50 smithi053 ceph-mon[23721]: mgrmap e34: smithi053.ygzjut(active, since 7s), standbys: smithi110.rivldx 2024-09-13T05:31:50.343 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:50 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"}]: dispatch 2024-09-13T05:31:50.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 124407 10.80 MB/sec execute 164 sec latency 59.838 ms 2024-09-13T05:31:50.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:50 smithi110 ceph-mon[25804]: mgrmap e34: smithi053.ygzjut(active, since 7s), standbys: smithi110.rivldx 2024-09-13T05:31:50.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:50 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"}]: dispatch 2024-09-13T05:31:51.358 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:51 smithi053 ceph-mon[23721]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.1 MiB/s wr, 329 op/s 2024-09-13T05:31:51.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 124703 10.80 MB/sec execute 165 sec latency 53.528 ms 2024-09-13T05:31:51.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:51 smithi110 ceph-mon[25804]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.1 MiB/s wr, 329 op/s 2024-09-13T05:31:52.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 124992 10.77 MB/sec execute 166 sec latency 47.153 ms 2024-09-13T05:31:53.122 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:53 smithi053 ceph-mon[23721]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.4 MiB/s wr, 256 op/s 2024-09-13T05:31:53.122 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:53 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:53.122 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:53 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:53.123 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:53 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:53.123 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:53 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:53.427 INFO:teuthology.orchestra.run.smithi053.stdout: 5 125268 10.74 MB/sec execute 167 sec latency 55.396 ms 2024-09-13T05:31:53.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:53 smithi110 ceph-mon[25804]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.4 MiB/s wr, 256 op/s 2024-09-13T05:31:53.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:53 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:53.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:53 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:53.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:53 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:53.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:53 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:54.430 INFO:teuthology.orchestra.run.smithi053.stdout: 5 125607 10.71 MB/sec execute 168 sec latency 51.001 ms 2024-09-13T05:31:55.164 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:55 smithi053 ceph-mon[23721]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.1 MiB/s wr, 401 op/s 2024-09-13T05:31:55.430 INFO:teuthology.orchestra.run.smithi053.stdout: 5 125927 10.69 MB/sec execute 169 sec latency 81.221 ms 2024-09-13T05:31:55.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:55 smithi110 ceph-mon[25804]: pgmap v9: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.1 MiB/s wr, 401 op/s 2024-09-13T05:31:56.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 126366 10.73 MB/sec execute 170 sec latency 32.946 ms 2024-09-13T05:31:57.186 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:57 smithi053 ceph-mon[23721]: pgmap v10: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.8 MiB/s wr, 367 op/s 2024-09-13T05:31:57.430 INFO:teuthology.orchestra.run.smithi053.stdout: 5 126808 10.72 MB/sec execute 171 sec latency 44.399 ms 2024-09-13T05:31:57.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:57 smithi110 ceph-mon[25804]: pgmap v10: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.8 MiB/s wr, 367 op/s 2024-09-13T05:31:58.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 127144 10.69 MB/sec execute 172 sec latency 36.867 ms 2024-09-13T05:31:58.963 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:31:59.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:59 smithi053 ceph-mon[23721]: pgmap v11: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.9 MiB/s wr, 375 op/s 2024-09-13T05:31:59.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:59 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:59.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:59 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:59.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:59 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:59.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:59 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:31:59.212 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:31:59 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:31:59.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 127399 10.68 MB/sec execute 173 sec latency 74.116 ms 2024-09-13T05:31:59.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:59 smithi110 ceph-mon[25804]: pgmap v11: 97 pgs: 97 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.9 MiB/s wr, 375 op/s 2024-09-13T05:31:59.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:59 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:31:59.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:59 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:59.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:59 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:31:59.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:59 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:31:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:31:59 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:31:59.887 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:31:59.887 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (88s) 7s ago 11m 19.9M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:31:59.887 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (11m) 7s ago 11m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:31:59.887 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (10m) 11s ago 10m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:31:59.887 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (79s) 7s ago 11m 44.9M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (5m) 7s ago 5m 36.7M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (5m) 11s ago 5m 15.2M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (35s) 7s ago 12m 492M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283 running (2m) 11s ago 10m 155M - 19.3.0-4911-g9d608eb1 0ce38589d192 e879e7d7efd2 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (12m) 7s ago 12m 55.1M 2048M 17.2.0 e1d6a67b021e 5d17dc93b2a5 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (10m) 11s ago 10m 44.0M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (5m) 7s ago 5m 269M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (5m) 11s ago 5m 89.4M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (72s) 7s ago 11m 10.0M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (44s) 11s ago 10m 9935k - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (9m) 7s ago 9m 461M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (9m) 7s ago 9m 396M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (8m) 7s ago 8m 293M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:31:59.888 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (8m) 7s ago 8m 422M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:31:59.889 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (8m) 11s ago 8m 421M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:31:59.889 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (7m) 11s ago 7m 466M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:31:59.889 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (7m) 11s ago 7m 408M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:31:59.889 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (6m) 11s ago 6m 550M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:31:59.889 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (50s) 7s ago 10m 41.5M - 2.51.0 1d3b7f56885b 3bed5cb02bbe 2024-09-13T05:32:00.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 127662 10.67 MB/sec execute 174 sec latency 82.805 ms 2024-09-13T05:32:00.441 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:32:00.441 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:32:00.441 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T05:32:00.442 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:32:00.443 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:32:00.443 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:32:00.496 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:32:00.496 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:32:00.496 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:00.496 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:00.496 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:00.496 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:00.496 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:00 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1337219425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:32:00.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:32:00.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:32:00.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:00.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='client.14868 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:00.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:00.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:00.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:00.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:00 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1337219425' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:32:00.892 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:32:00.892 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:32:00.892 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "", 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:32:00.893 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:32:01.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 128001 10.67 MB/sec execute 175 sec latency 47.918 ms 2024-09-13T05:32:01.508 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:32:01.517 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:01 smithi053 ceph-mon[23721]: Reconfiguring prometheus.smithi053 (dependencies changed)... 2024-09-13T05:32:01.517 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:01 smithi053 ceph-mon[23721]: from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:01.517 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:01 smithi053 ceph-mon[23721]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-09-13T05:32:01.517 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:01 smithi053 ceph-mon[23721]: Reconfiguring daemon prometheus.smithi053 on smithi053 2024-09-13T05:32:01.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:01 smithi110 ceph-mon[25804]: Reconfiguring prometheus.smithi053 (dependencies changed)... 2024-09-13T05:32:01.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:01 smithi110 ceph-mon[25804]: from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:01.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:01 smithi110 ceph-mon[25804]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 5.8 MiB/s wr, 588 op/s 2024-09-13T05:32:01.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:01 smithi110 ceph-mon[25804]: Reconfiguring daemon prometheus.smithi053 on smithi053 2024-09-13T05:32:02.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 128294 10.67 MB/sec execute 176 sec latency 54.218 ms 2024-09-13T05:32:02.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:02 smithi053 ceph-mon[23721]: from='client.14884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:02.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:02 smithi053 ceph-mon[23721]: from='client.? 172.21.15.53:0/1758617255' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:32:02.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:02 smithi110 ceph-mon[25804]: from='client.14884 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:02.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:02 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1758617255' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:32:03.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 128614 10.65 MB/sec execute 177 sec latency 55.760 ms 2024-09-13T05:32:03.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:03 smithi053 ceph-mon[23721]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-13T05:32:03.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:03 smithi110 ceph-mon[25804]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-13T05:32:04.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 128839 10.62 MB/sec execute 178 sec latency 73.534 ms 2024-09-13T05:32:05.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 129212 10.59 MB/sec execute 179 sec latency 53.402 ms 2024-09-13T05:32:05.679 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:05 smithi053 ceph-mon[23721]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-09-13T05:32:05.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:05 smithi110 ceph-mon[25804]: pgmap v14: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 578 op/s 2024-09-13T05:32:06.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 129509 10.57 MB/sec execute 180 sec latency 70.472 ms 2024-09-13T05:32:07.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 129888 10.60 MB/sec execute 181 sec latency 65.534 ms 2024-09-13T05:32:07.490 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:07 smithi053 ceph-mon[23721]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-13T05:32:07.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:07 smithi110 ceph-mon[25804]: pgmap v15: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-13T05:32:08.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 130250 10.58 MB/sec execute 182 sec latency 50.454 ms 2024-09-13T05:32:08.782 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:08 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:08.782 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:08 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:08.782 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:08 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:32:08.782 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:08 smithi053 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:32:08.782 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:08 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:32:08.877 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:08 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:08.878 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:08 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:08.878 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:08 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:32:08.878 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:08 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T05:32:08.878 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:08 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:32:09.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 130550 10.56 MB/sec execute 183 sec latency 67.939 ms 2024-09-13T05:32:09.807 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:09 smithi053 ceph-mon[23721]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-13T05:32:09.882 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:09 smithi110 ceph-mon[25804]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-13T05:32:10.431 INFO:teuthology.orchestra.run.smithi053.stdout: 5 130797 10.53 MB/sec execute 184 sec latency 60.447 ms 2024-09-13T05:32:10.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:10 smithi053 ceph-mon[23721]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 596 op/s 2024-09-13T05:32:10.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:10 smithi053 ceph-mon[23721]: Upgrade: Updating mgr.smithi110.rivldx 2024-09-13T05:32:10.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:10 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:32:10.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:10 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:32:10.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:10 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:32:10.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:10 smithi053 ceph-mon[23721]: Deploying daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:32:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:10 smithi110 ceph-mon[25804]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 596 op/s 2024-09-13T05:32:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:10 smithi110 ceph-mon[25804]: Upgrade: Updating mgr.smithi110.rivldx 2024-09-13T05:32:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:10 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T05:32:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:10 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T05:32:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:10 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:32:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:10 smithi110 ceph-mon[25804]: Deploying daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:32:11.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 131071 10.53 MB/sec execute 185 sec latency 45.465 ms 2024-09-13T05:32:12.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 131308 10.52 MB/sec execute 186 sec latency 71.486 ms 2024-09-13T05:32:13.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:13 smithi110 ceph-mon[25804]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 382 op/s 2024-09-13T05:32:13.281 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:13 smithi053 ceph-mon[23721]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 382 op/s 2024-09-13T05:32:13.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 131559 10.50 MB/sec execute 187 sec latency 89.637 ms 2024-09-13T05:32:14.304 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:14 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:32:14.342 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:14 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:32:14.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 131853 10.50 MB/sec execute 188 sec latency 61.849 ms 2024-09-13T05:32:15.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 132146 10.47 MB/sec execute 189 sec latency 51.296 ms 2024-09-13T05:32:15.579 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:15 smithi053 ceph-mon[23721]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.7 MiB/s wr, 545 op/s 2024-09-13T05:32:15.579 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:15 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:15.579 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:15 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:15.579 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:15 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:32:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:15 smithi110 ceph-mon[25804]: pgmap v19: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.7 MiB/s wr, 545 op/s 2024-09-13T05:32:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:15 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:15 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:15 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:32:16.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 132384 10.45 MB/sec execute 190 sec latency 76.654 ms 2024-09-13T05:32:17.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 132661 10.42 MB/sec execute 191 sec latency 50.299 ms 2024-09-13T05:32:17.530 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:17 smithi053 ceph-mon[23721]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-09-13T05:32:17.692 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:17 smithi110 ceph-mon[25804]: pgmap v20: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-09-13T05:32:18.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 133002 10.40 MB/sec execute 192 sec latency 54.242 ms 2024-09-13T05:32:19.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 133300 10.40 MB/sec execute 193 sec latency 77.809 ms 2024-09-13T05:32:19.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:19 smithi053 ceph-mon[23721]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-13T05:32:19.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:19 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:19.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:19 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:19.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:19 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:19.465 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:19 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:19 smithi110 ceph-mon[25804]: pgmap v21: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.8 MiB/s wr, 373 op/s 2024-09-13T05:32:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:19 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:19 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:19 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:19.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:19 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:20.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 133588 10.40 MB/sec execute 194 sec latency 56.785 ms 2024-09-13T05:32:20.753 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:20 smithi053 ceph-mon[23721]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 532 op/s 2024-09-13T05:32:20.939 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:20 smithi110 ceph-mon[25804]: pgmap v22: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 532 op/s 2024-09-13T05:32:21.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 133929 10.38 MB/sec execute 195 sec latency 56.697 ms 2024-09-13T05:32:21.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:21 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:32:21.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:21 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:32:21.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:21 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:32:21.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:21 smithi053 ceph-mon[23721]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:32:21.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:21 smithi053 ceph-mon[23721]: Standby manager daemon smithi110.rivldx restarted 2024-09-13T05:32:21.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:21 smithi053 ceph-mon[23721]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:32:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:21 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:32:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:21 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:32:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:21 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:32:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:21 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/4103115594' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:32:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:21 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx restarted 2024-09-13T05:32:21.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:21 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:32:22.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 134214 10.36 MB/sec execute 196 sec latency 38.638 ms 2024-09-13T05:32:22.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:22 smithi053 ceph-mon[23721]: mgrmap e35: smithi053.ygzjut(active, since 40s), standbys: smithi110.rivldx 2024-09-13T05:32:22.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:22 smithi053 ceph-mon[23721]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 338 op/s 2024-09-13T05:32:22.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:22 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:22.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:22 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:22 smithi110 ceph-mon[25804]: mgrmap e35: smithi053.ygzjut(active, since 40s), standbys: smithi110.rivldx 2024-09-13T05:32:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:22 smithi110 ceph-mon[25804]: pgmap v23: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 338 op/s 2024-09-13T05:32:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:22 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:22 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:23.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 134486 10.35 MB/sec execute 197 sec latency 74.534 ms 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:32:23.893 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:32:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: Upgrade: Setting container_image for all mgr 2024-09-13T05:32:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi053.ygzjut"}]: dispatch 2024-09-13T05:32:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi053.ygzjut"}]': finished 2024-09-13T05:32:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi110.rivldx"}]: dispatch 2024-09-13T05:32:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi110.rivldx"}]': finished 2024-09-13T05:32:23.894 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:23 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T05:32:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:32:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T05:32:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:24.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T05:32:24.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:32:24.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: Upgrade: Setting container_image for all mgr 2024-09-13T05:32:24.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:24.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi053.ygzjut"}]: dispatch 2024-09-13T05:32:24.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi053.ygzjut"}]': finished 2024-09-13T05:32:24.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi110.rivldx"}]: dispatch 2024-09-13T05:32:24.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi110.rivldx"}]': finished 2024-09-13T05:32:24.246 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:23 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T05:32:24.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 134742 10.34 MB/sec execute 198 sec latency 86.234 ms 2024-09-13T05:32:24.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:24 smithi053 ceph-mon[23721]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 524 op/s 2024-09-13T05:32:25.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:24 smithi110 ceph-mon[25804]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 524 op/s 2024-09-13T05:32:25.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 135341 10.39 MB/sec execute 199 sec latency 86.155 ms 2024-09-13T05:32:26.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 135757 10.39 MB/sec execute 200 sec latency 45.115 ms 2024-09-13T05:32:26.924 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:26 smithi053 ceph-mon[23721]: Upgrade: Updating mon.smithi053 2024-09-13T05:32:26.924 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:26 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:26.924 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:26 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:32:26.924 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:26 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T05:32:26.924 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:26 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:32:26.925 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:26 smithi053 ceph-mon[23721]: Deploying daemon mon.smithi053 on smithi053 2024-09-13T05:32:26.925 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:26 smithi053 ceph-mon[23721]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 361 op/s 2024-09-13T05:32:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:26 smithi110 ceph-mon[25804]: Upgrade: Updating mon.smithi053 2024-09-13T05:32:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:26 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:26 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T05:32:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:26 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T05:32:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:26 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T05:32:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:26 smithi110 ceph-mon[25804]: Deploying daemon mon.smithi053 on smithi053 2024-09-13T05:32:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:26 smithi110 ceph-mon[25804]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 361 op/s 2024-09-13T05:32:27.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 136068 10.36 MB/sec execute 201 sec latency 61.737 ms 2024-09-13T05:32:28.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 136391 10.35 MB/sec execute 202 sec latency 63.142 ms 2024-09-13T05:32:28.758 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:28 smithi053 systemd[1]: Stopping Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:32:29.268 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:29 smithi053 ceph-mon[23721]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.1 MiB/s wr, 448 op/s 2024-09-13T05:32:29.268 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:29 smithi053 ceph-mon[23721]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:32:29.268 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:29 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[23717]: 2024-09-13T05:32:29.102+0000 7fdeaac28700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi053 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:32:29.268 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:29 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[23717]: 2024-09-13T05:32:29.102+0000 7fdeaac28700 -1 mon.smithi053@0(leader) e2 *** Got Signal Terminated *** 2024-09-13T05:32:29.268 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:29 smithi053 podman[64078]: 2024-09-13 05:32:29.136638134 +0000 UTC m=+0.200287547 container died 5d17dc93b2a5b1c4e65e307cbb786562415ef5e905001f3133b4067443c57822 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, version=8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, ceph=True) 2024-09-13T05:32:29.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 136608 10.32 MB/sec execute 203 sec latency 62.587 ms 2024-09-13T05:32:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:29 smithi110 ceph-mon[25804]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.1 MiB/s wr, 448 op/s 2024-09-13T05:32:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:29 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T05:32:30.032 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:30 smithi053 bash[64078]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053 2024-09-13T05:32:30.032 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:30 smithi053 podman[64078]: 2024-09-13 05:32:30.013605619 +0000 UTC m=+1.077255024 container cleanup 5d17dc93b2a5b1c4e65e307cbb786562415ef5e905001f3133b4067443c57822 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, name=centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.expose-services=, io.buildah.version=1.19.8, vcs-type=git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD) 2024-09-13T05:32:30.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 136900 10.34 MB/sec execute 204 sec latency 50.882 ms 2024-09-13T05:32:30.570 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:30 smithi053 podman[64089]: 2024-09-13 05:32:30.537071924 +0000 UTC m=+1.396187200 container remove 5d17dc93b2a5b1c4e65e307cbb786562415ef5e905001f3133b4067443c57822 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.component=centos-stream-container, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-13T05:32:31.073 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:30 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service: Deactivated successfully. 2024-09-13T05:32:31.073 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:30 smithi053 systemd[1]: Stopped Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:32:31.073 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:30 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service: Consumed 18.278s CPU time. 2024-09-13T05:32:31.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 137339 10.33 MB/sec execute 205 sec latency 51.153 ms 2024-09-13T05:32:31.575 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:31 smithi053 systemd[1]: Starting Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:32:31.827 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:31 smithi053 podman[64223]: 2024-09-13 05:32:31.518784704 +0000 UTC m=+0.024589434 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:32:31.827 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:31 smithi053 podman[64223]: 2024-09-13 05:32:31.695643741 +0000 UTC m=+0.201448475 container create 7ce01e261ca36ff49f719c8ab239a4b8cc930ea8fac1a903925cec914d05db36 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:32:32.161 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:32:32.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 137666 10.31 MB/sec execute 206 sec latency 51.540 ms 2024-09-13T05:32:32.582 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 podman[64223]: 2024-09-13 05:32:32.393662755 +0000 UTC m=+0.899467495 container init 7ce01e261ca36ff49f719c8ab239a4b8cc930ea8fac1a903925cec914d05db36 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 podman[64223]: 2024-09-13 05:32:32.399544346 +0000 UTC m=+0.905349080 container start 7ce01e261ca36ff49f719c8ab239a4b8cc930ea8fac1a903925cec914d05db36 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev), process ceph-mon, pid 2 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: pidfile_write: ignore empty --pid-file 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: load: jerasure load: lrc 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: RocksDB version: 7.9.2 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Git sha 0 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Compile date 2024-09-12 17:11:01 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: DB SUMMARY 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: DB Session ID: XQLGQA2ZHYDGZ7ILAH3Q 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: CURRENT file: CURRENT 2024-09-13T05:32:32.583 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: MANIFEST file: MANIFEST-000015 size: 971 Bytes 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi053/store.db dir, Total Num: 1, files: 000030.sst 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi053/store.db: 000028.log size: 3417283 ; 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.error_if_exists: 0 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.create_if_missing: 0 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.paranoid_checks: 1 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.env: 0x561ff64b5120 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T05:32:32.584 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.info_log: 0x561ff70938e0 2024-09-13T05:32:32.585 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.statistics: (nil) 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.use_fsync: 0 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_log_file_size: 0 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T05:32:32.586 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.allow_fallocate: 1 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.use_direct_reads: 0 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.db_log_dir: 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.wal_dir: 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T05:32:32.587 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.write_buffer_manager: 0x561ff7097b80 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T05:32:32.588 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.unordered_write: 0 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.row_cache: None 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.wal_filter: None 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.two_write_queues: 0 2024-09-13T05:32:32.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.wal_compression: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.atomic_flush: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.log_readahead_size: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T05:32:32.590 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_background_jobs: 2 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_background_compactions: -1 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_subcompactions: 1 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T05:32:32.591 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_open_files: -1 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_background_flushes: -1 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Compression algorithms supported: 2024-09-13T05:32:32.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kZSTD supported: 0 2024-09-13T05:32:32.593 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kXpressCompression supported: 0 2024-09-13T05:32:32.593 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kBZip2Compression supported: 0 2024-09-13T05:32:32.593 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T05:32:32.593 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kLZ4Compression supported: 1 2024-09-13T05:32:32.593 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kZlibCompression supported: 1 2024-09-13T05:32:32.593 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T05:32:32.593 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: kSnappyCompression supported: 1 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi053/store.db/MANIFEST-000015 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.merge_operator: 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_filter: None 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T05:32:32.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561ff7093540) 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: cache_index_and_filter_blocks: 1 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: pin_top_level_index_and_filter: 1 2024-09-13T05:32:32.595 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: index_type: 0 2024-09-13T05:32:32.596 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: data_block_index_type: 0 2024-09-13T05:32:32.596 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: index_shortening: 1 2024-09-13T05:32:32.596 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T05:32:32.596 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: checksum: 4 2024-09-13T05:32:32.596 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: no_block_cache: 0 2024-09-13T05:32:32.596 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache: 0x561ff7087610 2024-09-13T05:32:32.596 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache_name: BinnedLRUCache 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache_options: 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: capacity : 536870912 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: num_shard_bits : 4 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: strict_capacity_limit : 0 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: high_pri_pool_ratio: 0.000 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_cache_compressed: (nil) 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: persistent_cache: (nil) 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_size: 4096 2024-09-13T05:32:32.597 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_size_deviation: 10 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_restart_interval: 16 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: index_block_restart_interval: 1 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: metadata_block_size: 4096 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: partition_filters: 0 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: use_delta_encoding: 1 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: filter_policy: bloomfilter 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: whole_key_filtering: 1 2024-09-13T05:32:32.598 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: verify_compression: 0 2024-09-13T05:32:32.599 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: read_amp_bytes_per_bit: 0 2024-09-13T05:32:32.599 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: format_version: 5 2024-09-13T05:32:32.599 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: enable_index_compression: 1 2024-09-13T05:32:32.599 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: block_align: 0 2024-09-13T05:32:32.599 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: max_auto_readahead_size: 262144 2024-09-13T05:32:32.599 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: prepopulate_block_cache: 0 2024-09-13T05:32:32.599 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: initial_auto_readahead_size: 8192 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression: NoCompression 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T05:32:32.600 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.num_levels: 7 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T05:32:32.601 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T05:32:32.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T05:32:32.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T05:32:32.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T05:32:32.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T05:32:32.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T05:32:32.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T05:32:32.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T05:32:32.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T05:32:32.604 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T05:32:32.605 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T05:32:32.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.inplace_update_support: 0 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T05:32:32.607 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.bloom_locality: 0 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.max_successive_merges: 0 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T05:32:32.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.ttl: 2592000 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.enable_blob_files: false 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.min_blob_size: 0 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T05:32:32.609 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi053/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11773, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 55e9e85d-c981-474e-8a47-c0b879431a35 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726205552438953, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726205552455339, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 3127661, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11774, "largest_seqno": 13274, "table_properties": {"data_size": 3120502, "index_size": 4514, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1605, "raw_key_size": 15035, "raw_average_key_size": 23, "raw_value_size": 3106407, "raw_average_value_size": 4868, "num_data_blocks": 206, "num_entries": 638, "num_filter_entries": 638, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726205552, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "55e9e85d-c981-474e-8a47-c0b879431a35", "db_session_id": "XQLGQA2ZHYDGZ7ILAH3Q", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726205552455543, "job": 1, "event": "recovery_finished"} 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T05:32:32.610 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi053/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561ff70b6e00 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: DB pointer 0x561ff71b8000 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** DB Stats ** 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T05:32:32.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** Compaction Stats [default] ** 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T05:32:32.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: L0 1/0 2.98 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 395.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: L6 1/0 6.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Sum 2/0 9.20 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 395.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 395.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** Compaction Stats [default] ** 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 395.1 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.613 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:32:32.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-13T05:32:32.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T05:32:32.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T05:32:32.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T05:32:32.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T05:32:32.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Cumulative compaction: 0.00 GB write, 146.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:32:32.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Interval compaction: 0.00 GB write, 146.47 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:32:32.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T05:32:32.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Block cache BinnedLRUCache@0x561ff7087610#2 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-13T05:32:32.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2024-09-13T05:32:32.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: starting mon.smithi053 rank 0 at public addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] at bind addrs [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi053 fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???) e2 preinit fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???).mds e7 new map 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???).mds e7 print_map 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: e7 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T05:32:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: legacy client fscid: 1 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Filesystem 'foofs' (1) 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: fs_name foofs 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: epoch 5 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: created 2024-09-13T05:26:28.810720+0000 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: modified 2024-09-13T05:26:33.825671+0000 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: tableserver 0 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: root 0 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: session_timeout 60 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: session_autoclose 300 2024-09-13T05:32:32.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: max_file_size 1099511627776 2024-09-13T05:32:32.618 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: max_xattr_size 65536 2024-09-13T05:32:32.618 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: required_client_features {} 2024-09-13T05:32:32.618 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: last_failure 0 2024-09-13T05:32:32.618 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: last_failure_osd_epoch 0 2024-09-13T05:32:32.618 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T05:32:32.618 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: max_mds 1 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: in 0 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: up {0=14506} 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: failed 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: damaged 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: stopped 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: data_pools [3] 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: metadata_pool 2 2024-09-13T05:32:32.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: inline_data disabled 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: balancer 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: bal_rank_mask -1 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: standby_count_wanted 1 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: qdb_cluster leader: 0 members: 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: [mds.foofs.smithi053.rzvrng{0:14506} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.53:6834/2495430984,v1:172.21.15.53:6835/2495430984] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Standby daemons: 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: [mds.foofs.smithi110.akbygi{-1:24313} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 ceph-mon[64292]: mon.smithi053@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 bash[64223]: 7ce01e261ca36ff49f719c8ab239a4b8cc930ea8fac1a903925cec914d05db36 2024-09-13T05:32:32.621 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:32 smithi053 systemd[1]: Started Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (2m) 11s ago 12m 19.9M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (11m) 11s ago 11m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (11m) 14s ago 11m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (113s) 11s ago 11m 43.1M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (6m) 11s ago 6m 36.6M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (5m) 14s ago 5m 15.3M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:32:33.362 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (69s) 11s ago 13m 508M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (19s) 14s ago 11m 66.4M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 starting - - - 2048M 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (11m) 14s ago 11m 43.6M 2048M 17.2.0 e1d6a67b021e 2229074820b4 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (5m) 11s ago 5m 278M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (5m) 14s ago 5m 89.4M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (105s) 11s ago 11m 9.98M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (78s) 14s ago 11m 9.80M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (10m) 11s ago 10m 490M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (10m) 11s ago 10m 401M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (9m) 11s ago 9m 299M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (9m) 11s ago 9m 447M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:32:33.363 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (8m) 14s ago 8m 471M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:32:33.364 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (8m) 14s ago 8m 515M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:32:33.364 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (7m) 14s ago 7m 423M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:32:33.364 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (7m) 14s ago 7m 329M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:32:33.364 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (26s) 11s ago 10m 43.8M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:32:33.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 137998 10.31 MB/sec execute 207 sec latency 93.454 ms 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:33.950 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:32:33.951 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:32:34.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 138193 10.29 MB/sec execute 208 sec latency 224.469 ms 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:32:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T05:32:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T05:32:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:32:34.479 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:32:34.906 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:32:35.432 INFO:teuthology.orchestra.run.smithi053.stdout: 5 138193 10.24 MB/sec execute 209 sec latency 1224.519 ms 2024-09-13T05:32:35.818 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.6 MiB/s wr, 577 op/s 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='client.24613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='client.24621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/263925458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: pgmap v29: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-09-13T05:32:35.819 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='client.24629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1535491108' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: mon.smithi053 calling monitor election 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: mon.smithi053 is new leader, mons smithi053,smithi110 in quorum (ranks 0,1) 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: monmap epoch 2 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: last_changed 2024-09-13T05:21:17.695187+0000 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: created 2024-09-13T05:19:06.284452+0000 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: min_mon_release 17 (quincy) 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: election_strategy: 1 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: 0: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.smithi053 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: 1: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.smithi110 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:32:35.820 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T05:32:35.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: mgrmap e35: smithi053.ygzjut(active, since 54s), standbys: smithi110.rivldx 2024-09-13T05:32:35.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: overall HEALTH_OK 2024-09-13T05:32:35.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: from='mgr.14824 ' entity='' 2024-09-13T05:32:35.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:35 smithi053 ceph-mon[64292]: mgrmap e36: smithi053.ygzjut(active, since 54s), standbys: smithi110.rivldx 2024-09-13T05:32:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi053"}]: dispatch 2024-09-13T05:32:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "mon metadata", "id": "smithi110"}]: dispatch 2024-09-13T05:32:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='mgr.14824 172.21.15.53:0/3692094898' entity='mgr.smithi053.ygzjut' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.6 MiB/s wr, 577 op/s 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='client.24613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='client.24617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='client.24621 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/263925458' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: pgmap v29: 97 pgs: 97 active+clean; 333 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.9 MiB/s wr, 604 op/s 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='client.24629 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='client.? 172.21.15.53:0/1535491108' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: mon.smithi053 calling monitor election 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: mon.smithi053 is new leader, mons smithi053,smithi110 in quorum (ranks 0,1) 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: monmap epoch 2 2024-09-13T05:32:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: last_changed 2024-09-13T05:21:17.695187+0000 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: created 2024-09-13T05:19:06.284452+0000 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: min_mon_release 17 (quincy) 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: election_strategy: 1 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: 0: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.smithi053 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: 1: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.smithi110 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: mgrmap e35: smithi053.ygzjut(active, since 54s), standbys: smithi110.rivldx 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: overall HEALTH_OK 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: from='mgr.14824 ' entity='' 2024-09-13T05:32:35.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:35 smithi110 ceph-mon[25804]: mgrmap e36: smithi053.ygzjut(active, since 54s), standbys: smithi110.rivldx 2024-09-13T05:32:36.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 138454 10.23 MB/sec execute 210 sec latency 1338.159 ms 2024-09-13T05:32:37.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 138813 10.24 MB/sec execute 211 sec latency 57.030 ms 2024-09-13T05:32:37.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:37 smithi053 ceph-mon[64292]: mgrmap e37: smithi053.ygzjut(active, since 55s), standbys: smithi110.rivldx 2024-09-13T05:32:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:37 smithi110 ceph-mon[25804]: mgrmap e37: smithi053.ygzjut(active, since 55s), standbys: smithi110.rivldx 2024-09-13T05:32:38.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 139001 10.21 MB/sec execute 212 sec latency 88.393 ms 2024-09-13T05:32:39.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 139235 10.20 MB/sec execute 213 sec latency 68.887 ms 2024-09-13T05:32:40.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 139361 10.17 MB/sec execute 214 sec latency 72.806 ms 2024-09-13T05:32:41.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 139568 10.13 MB/sec execute 215 sec latency 53.305 ms 2024-09-13T05:32:41.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:41 smithi053 ceph-mon[64292]: Standby manager daemon smithi110.rivldx restarted 2024-09-13T05:32:41.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:41 smithi053 ceph-mon[64292]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:32:41.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:41 smithi053 ceph-mon[64292]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:32:41.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:41 smithi053 ceph-mon[64292]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:32:41.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:41 smithi053 ceph-mon[64292]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:32:41.478 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:41 smithi053 ceph-mon[64292]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:32:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:41 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx restarted 2024-09-13T05:32:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:41 smithi110 ceph-mon[25804]: Standby manager daemon smithi110.rivldx started 2024-09-13T05:32:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:41 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/crt"}]: dispatch 2024-09-13T05:32:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:41 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T05:32:41.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:41 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi110.rivldx/key"}]: dispatch 2024-09-13T05:32:41.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:41 smithi110 ceph-mon[25804]: from='mgr.? 172.21.15.110:0/2823696991' entity='mgr.smithi110.rivldx' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T05:32:42.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 139782 10.11 MB/sec execute 216 sec latency 92.055 ms 2024-09-13T05:32:42.510 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: mgrmap e38: smithi053.ygzjut(active, since 60s), standbys: smithi110.rivldx 2024-09-13T05:32:42.510 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: Active manager daemon smithi053.ygzjut restarted 2024-09-13T05:32:42.510 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: Activating manager daemon smithi053.ygzjut 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: mgrmap e39: smithi053.ygzjut(active, starting, since 0.0328546s), standbys: smithi110.rivldx 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi053"} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi110"} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T05:32:42.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T05:32:42.512 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T05:32:42.512 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T05:32:42.512 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T05:32:42.512 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T05:32:42.512 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T05:32:42.512 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T05:32:42.512 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:42 smithi053 ceph-mon[64292]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:32:42.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: mgrmap e38: smithi053.ygzjut(active, since 60s), standbys: smithi110.rivldx 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: Active manager daemon smithi053.ygzjut restarted 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: Activating manager daemon smithi053.ygzjut 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: mgrmap e39: smithi053.ygzjut(active, starting, since 0.0328546s), standbys: smithi110.rivldx 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi053"} : dispatch 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi110"} : dispatch 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"} : dispatch 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr metadata", "who": "smithi053.ygzjut", "id": "smithi053.ygzjut"} : dispatch 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr metadata", "who": "smithi110.rivldx", "id": "smithi110.rivldx"} : dispatch 2024-09-13T05:32:42.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T05:32:42.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:42 smithi110 ceph-mon[25804]: Manager daemon smithi053.ygzjut is now available 2024-09-13T05:32:43.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 140064 10.09 MB/sec execute 217 sec latency 62.813 ms 2024-09-13T05:32:43.541 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:32:43.541 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T05:32:43.541 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:32:43.541 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"} : dispatch 2024-09-13T05:32:43.541 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"} : dispatch 2024-09-13T05:32:43.541 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:43 smithi053 ceph-mon[64292]: mgrmap e40: smithi053.ygzjut(active, since 1.03946s), standbys: smithi110.rivldx 2024-09-13T05:32:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:43 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:32:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:43 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T05:32:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:43 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:32:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:43 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/mirror_snapshot_schedule"} : dispatch 2024-09-13T05:32:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:43 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi053.ygzjut/trash_purge_schedule"} : dispatch 2024-09-13T05:32:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:43 smithi110 ceph-mon[25804]: mgrmap e40: smithi053.ygzjut(active, since 1.03946s), standbys: smithi110.rivldx 2024-09-13T05:32:44.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 140310 10.08 MB/sec execute 218 sec latency 73.145 ms 2024-09-13T05:32:44.563 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:44 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:32:44.563 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:44 smithi053 ceph-mon[64292]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:32:44.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:44 smithi110 ceph-mon[25804]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T05:32:44.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:44 smithi110 ceph-mon[25804]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:32:45.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 140788 10.09 MB/sec execute 219 sec latency 33.052 ms 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:45 smithi053 ceph-mon[64292]: [13/Sep/2024:05:32:44] ENGINE Bus STARTING 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:45 smithi053 ceph-mon[64292]: [13/Sep/2024:05:32:44] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:45 smithi053 ceph-mon[64292]: [13/Sep/2024:05:32:44] ENGINE Error in HTTPServer.serve 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: Traceback (most recent call last): 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._connections.run(self.expiration_interval) 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._run(expiration_interval) 2024-09-13T05:32:45.571 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: s = self.context.wrap_socket( 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: return self.sslsocket_class._create( 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self.do_handshake() 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: self._sslobj.do_handshake() 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:45 smithi053 ceph-mon[64292]: [13/Sep/2024:05:32:44] ENGINE Serving on http://172.21.15.53:8765 2024-09-13T05:32:45.572 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:45 smithi053 ceph-mon[64292]: [13/Sep/2024:05:32:44] ENGINE Bus STARTED 2024-09-13T05:32:45.573 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:45 smithi053 ceph-mon[64292]: mgrmap e41: smithi053.ygzjut(active, since 2s), standbys: smithi110.rivldx 2024-09-13T05:32:45.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:45 smithi110 ceph-mon[25804]: [13/Sep/2024:05:32:44] ENGINE Bus STARTING 2024-09-13T05:32:45.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:45 smithi110 ceph-mon[25804]: [13/Sep/2024:05:32:44] ENGINE Serving on https://172.21.15.53:7150 2024-09-13T05:32:45.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:45 smithi110 ceph-mon[25804]: [13/Sep/2024:05:32:44] ENGINE Error in HTTPServer.serve 2024-09-13T05:32:45.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Traceback (most recent call last): 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._connections.run(self.expiration_interval) 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._run(expiration_interval) 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: s = self.context.wrap_socket( 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: return self.sslsocket_class._create( 2024-09-13T05:32:45.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T05:32:45.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self.do_handshake() 2024-09-13T05:32:45.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T05:32:45.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: self._sslobj.do_handshake() 2024-09-13T05:32:45.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T05:32:45.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:45 smithi110 ceph-mon[25804]: [13/Sep/2024:05:32:44] ENGINE Serving on http://172.21.15.53:8765 2024-09-13T05:32:45.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:45 smithi110 ceph-mon[25804]: [13/Sep/2024:05:32:44] ENGINE Bus STARTED 2024-09-13T05:32:45.746 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:45 smithi110 ceph-mon[25804]: mgrmap e41: smithi053.ygzjut(active, since 2s), standbys: smithi110.rivldx 2024-09-13T05:32:46.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 141267 10.09 MB/sec execute 220 sec latency 34.787 ms 2024-09-13T05:32:47.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 141591 10.09 MB/sec execute 221 sec latency 63.809 ms 2024-09-13T05:32:47.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:47 smithi053 ceph-mon[64292]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:32:47.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:47 smithi053 ceph-mon[64292]: mgrmap e42: smithi053.ygzjut(active, since 4s), standbys: smithi110.rivldx 2024-09-13T05:32:47.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:47 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:47.608 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:47 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:47 smithi110 ceph-mon[25804]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:32:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:47 smithi110 ceph-mon[25804]: mgrmap e42: smithi053.ygzjut(active, since 4s), standbys: smithi110.rivldx 2024-09-13T05:32:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:47 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:47 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:48.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 142036 10.11 MB/sec execute 222 sec latency 44.507 ms 2024-09-13T05:32:48.903 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:48.903 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:48.903 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:48 smithi053 ceph-mon[64292]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:32:49.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:48 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:49.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:48 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:49.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:48 smithi110 ceph-mon[25804]: pgmap v6: 97 pgs: 97 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T05:32:49.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 142353 10.12 MB/sec execute 223 sec latency 63.693 ms 2024-09-13T05:32:50.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 142637 10.10 MB/sec execute 224 sec latency 57.209 ms 2024-09-13T05:32:51.192 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:51 smithi053 ceph-mon[64292]: pgmap v7: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 401 op/s 2024-09-13T05:32:51.232 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:51 smithi110 ceph-mon[25804]: pgmap v7: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 401 op/s 2024-09-13T05:32:51.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 142811 10.08 MB/sec execute 225 sec latency 58.968 ms 2024-09-13T05:32:52.433 INFO:teuthology.orchestra.run.smithi053.stdout: 5 143037 10.06 MB/sec execute 226 sec latency 53.619 ms 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:32:52.717 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:32:52.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:52.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:32:52.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:52 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:32:53.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 143292 10.04 MB/sec execute 227 sec latency 68.290 ms 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: pgmap v8: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.2 MiB/s wr, 311 op/s 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.734 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.735 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.735 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.735 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: pgmap v8: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.2 MiB/s wr, 311 op/s 2024-09-13T05:32:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.conf 2024-09-13T05:32:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.conf 2024-09-13T05:32:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T05:32:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:53 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:54.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 143490 10.01 MB/sec execute 228 sec latency 64.767 ms 2024-09-13T05:32:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:54 smithi110 ceph-mon[25804]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:54 smithi110 ceph-mon[25804]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:54 smithi110 ceph-mon[25804]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:54 smithi110 ceph-mon[25804]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:54 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:32:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:54 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:32:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:54 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T05:32:54.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:54 smithi053 ceph-mon[64292]: Updating smithi110:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:54.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:54 smithi053 ceph-mon[64292]: Updating smithi053:/etc/ceph/ceph.client.admin.keyring 2024-09-13T05:32:54.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:54 smithi053 ceph-mon[64292]: Updating smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:54.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:54 smithi053 ceph-mon[64292]: Updating smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T05:32:54.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:54 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:32:54.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:54 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:32:54.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:54 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T05:32:55.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 143673 9.98 MB/sec execute 229 sec latency 61.421 ms 2024-09-13T05:32:55.804 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:55 smithi053 ceph-mon[64292]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.3 MiB/s wr, 434 op/s 2024-09-13T05:32:55.804 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:55 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:55.804 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:55 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T05:32:55.804 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:55 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T05:32:55.805 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:55 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:32:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:55 smithi110 ceph-mon[25804]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 4.3 MiB/s wr, 434 op/s 2024-09-13T05:32:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:55 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:32:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:55 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T05:32:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:55 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T05:32:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:55 smithi110 ceph-mon[25804]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:32:56.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 144004 9.99 MB/sec execute 230 sec latency 86.387 ms 2024-09-13T05:32:56.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:56 smithi053 ceph-mon[64292]: Upgrade: Updating mon.smithi110 2024-09-13T05:32:56.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:56 smithi053 ceph-mon[64292]: Deploying daemon mon.smithi110 on smithi110 2024-09-13T05:32:56.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:32:56 smithi053 ceph-mon[64292]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-13T05:32:56.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:56 smithi110 ceph-mon[25804]: Upgrade: Updating mon.smithi110 2024-09-13T05:32:56.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:56 smithi110 ceph-mon[25804]: Deploying daemon mon.smithi110 on smithi110 2024-09-13T05:32:56.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:56 smithi110 ceph-mon[25804]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 20 KiB/s rd, 3.9 MiB/s wr, 395 op/s 2024-09-13T05:32:57.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 144313 9.98 MB/sec execute 231 sec latency 75.666 ms 2024-09-13T05:32:57.579 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:57 smithi110 systemd[1]: Stopping Ceph mon.smithi110 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:32:57.962 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110[25798]: 2024-09-13T05:32:57.577+0000 7fe7e11a1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi110 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:32:57.963 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110[25798]: 2024-09-13T05:32:57.577+0000 7fe7e11a1700 -1 mon.smithi110@1(peon) e2 *** Got Signal Terminated *** 2024-09-13T05:32:57.963 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:57 smithi110 podman[49228]: 2024-09-13 05:32:57.649209593 +0000 UTC m=+0.191056800 container died 2229074820b4b7d642e036545c5bc0b3616f26acb24530d713e89889f14134b1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, release=754, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8) 2024-09-13T05:32:58.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 podman[49228]: 2024-09-13 05:32:58.045397948 +0000 UTC m=+0.587245154 container cleanup 2229074820b4b7d642e036545c5bc0b3616f26acb24530d713e89889f14134b1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, release=754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, version=8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T05:32:58.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 bash[49228]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110 2024-09-13T05:32:58.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 144713 9.98 MB/sec execute 232 sec latency 69.019 ms 2024-09-13T05:32:58.659 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 podman[49240]: 2024-09-13 05:32:58.289115972 +0000 UTC m=+0.641740711 container remove 2229074820b4b7d642e036545c5bc0b3616f26acb24530d713e89889f14134b1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, distribution-scope=public, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-type=git, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870) 2024-09-13T05:32:58.659 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi110.service: Deactivated successfully. 2024-09-13T05:32:58.660 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 systemd[1]: Stopped Ceph mon.smithi110 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:32:58.660 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi110.service: Consumed 7.006s CPU time. 2024-09-13T05:32:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 systemd[1]: Starting Ceph mon.smithi110 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:32:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 podman[49368]: 2024-09-13 05:32:58.778329129 +0000 UTC m=+0.017304820 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:32:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:58 smithi110 podman[49368]: 2024-09-13 05:32:58.907442207 +0000 UTC m=+0.146417892 container create 7c6115f393f971d7a7082dc8e15e6dbb19bf1583a3a06930882431c2e32f0321 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-13T05:32:59.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 145001 9.95 MB/sec execute 233 sec latency 71.450 ms 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 podman[49368]: 2024-09-13 05:32:59.135743981 +0000 UTC m=+0.374719673 container init 7c6115f393f971d7a7082dc8e15e6dbb19bf1583a3a06930882431c2e32f0321 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 podman[49368]: 2024-09-13 05:32:59.139232511 +0000 UTC m=+0.378208194 container start 7c6115f393f971d7a7082dc8e15e6dbb19bf1583a3a06930882431c2e32f0321 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev), process ceph-mon, pid 2 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: pidfile_write: ignore empty --pid-file 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: load: jerasure load: lrc 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: RocksDB version: 7.9.2 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Git sha 0 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Compile date 2024-09-12 17:11:01 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: DB SUMMARY 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: DB Session ID: H8AHBPFIVCRCD1YCBJ9I 2024-09-13T05:32:59.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: CURRENT file: CURRENT 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: MANIFEST file: MANIFEST-000009 size: 896 Bytes 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi110/store.db dir, Total Num: 1, files: 000024.sst 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi110/store.db: 000022.log size: 8601873 ; 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.error_if_exists: 0 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.create_if_missing: 0 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.paranoid_checks: 1 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.env: 0x55fad275d120 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.info_log: 0x55fad39a18e0 2024-09-13T05:32:59.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T05:32:59.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.statistics: (nil) 2024-09-13T05:32:59.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.use_fsync: 0 2024-09-13T05:32:59.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_log_file_size: 0 2024-09-13T05:32:59.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.allow_fallocate: 1 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.use_direct_reads: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.db_log_dir: 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.wal_dir: 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T05:32:59.498 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.write_buffer_manager: 0x55fad39a5b80 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T05:32:59.499 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.unordered_write: 0 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.row_cache: None 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.wal_filter: None 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.two_write_queues: 0 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.wal_compression: 0 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.atomic_flush: 0 2024-09-13T05:32:59.500 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.log_readahead_size: 0 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_background_jobs: 2 2024-09-13T05:32:59.501 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_background_compactions: -1 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_subcompactions: 1 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_open_files: -1 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T05:32:59.502 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_background_flushes: -1 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Compression algorithms supported: 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kZSTD supported: 0 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kXpressCompression supported: 0 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kBZip2Compression supported: 0 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kLZ4Compression supported: 1 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kZlibCompression supported: 1 2024-09-13T05:32:59.503 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: kSnappyCompression supported: 1 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi110/store.db/MANIFEST-000009 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.merge_operator: 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_filter: None 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fad39a1540) 2024-09-13T05:32:59.504 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: cache_index_and_filter_blocks: 1 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: pin_top_level_index_and_filter: 1 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: index_type: 0 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: data_block_index_type: 0 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: index_shortening: 1 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: checksum: 4 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: no_block_cache: 0 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache: 0x55fad3995610 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache_name: BinnedLRUCache 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache_options: 2024-09-13T05:32:59.505 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: capacity : 536870912 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: num_shard_bits : 4 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: strict_capacity_limit : 0 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: high_pri_pool_ratio: 0.000 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_cache_compressed: (nil) 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: persistent_cache: (nil) 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_size: 4096 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_size_deviation: 10 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_restart_interval: 16 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: index_block_restart_interval: 1 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: metadata_block_size: 4096 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: partition_filters: 0 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: use_delta_encoding: 1 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: filter_policy: bloomfilter 2024-09-13T05:32:59.506 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: whole_key_filtering: 1 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: verify_compression: 0 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: read_amp_bytes_per_bit: 0 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: format_version: 5 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: enable_index_compression: 1 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: block_align: 0 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: max_auto_readahead_size: 262144 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: prepopulate_block_cache: 0 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: initial_auto_readahead_size: 8192 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression: NoCompression 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T05:32:59.507 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.num_levels: 7 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T05:32:59.508 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T05:32:59.509 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T05:32:59.510 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T05:32:59.510 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T05:32:59.510 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T05:32:59.510 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T05:32:59.511 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.inplace_update_support: 0 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T05:32:59.512 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.bloom_locality: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.max_successive_merges: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.ttl: 2592000 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.enable_blob_files: false 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.min_blob_size: 0 2024-09-13T05:32:59.513 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi110/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12179, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 18392c62-3f6f-44aa-b1a2-80ac6a81e6fe 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726205579169002, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-13T05:32:59.514 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726205579191166, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 5230505, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12184, "largest_seqno": 14383, "table_properties": {"data_size": 5221076, "index_size": 6207, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2181, "raw_key_size": 20283, "raw_average_key_size": 23, "raw_value_size": 5202194, "raw_average_value_size": 6028, "num_data_blocks": 280, "num_entries": 863, "num_filter_entries": 863, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726205579, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "18392c62-3f6f-44aa-b1a2-80ac6a81e6fe", "db_session_id": "H8AHBPFIVCRCD1YCBJ9I", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726205579191293, "job": 1, "event": "recovery_finished"} 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi110/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55fad39c4e00 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: DB pointer 0x55fad3ac6000 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** DB Stats ** 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T05:32:59.515 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: L0 1/0 4.99 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 579.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: L6 1/0 6.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Sum 2/0 11.21 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 579.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 579.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** Compaction Stats [default] ** 2024-09-13T05:32:59.516 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 579.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Cumulative compaction: 0.00 GB write, 197.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Interval compaction: 0.00 GB write, 197.37 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T05:32:59.517 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Block cache BinnedLRUCache@0x55fad3995610#2 capacity: 512.00 MB usage: 8.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,6.58 KB,0.00125468%) Misc(1,0.00 KB,0%) 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: starting mon.smithi110 rank 1 at public addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] at bind addrs [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi110 fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???) e2 preinit fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???).mds e7 new map 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???).mds e7 print_map 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: e7 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: legacy client fscid: 1 2024-09-13T05:32:59.518 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Filesystem 'foofs' (1) 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: fs_name foofs 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: epoch 5 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: created 2024-09-13T05:26:28.810720+0000 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: modified 2024-09-13T05:26:33.825671+0000 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: tableserver 0 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: root 0 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: session_timeout 60 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: session_autoclose 300 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: max_file_size 1099511627776 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: max_xattr_size 65536 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: required_client_features {} 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: last_failure 0 2024-09-13T05:32:59.519 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: last_failure_osd_epoch 0 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: max_mds 1 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: in 0 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: up {0=14506} 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: failed 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: damaged 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: stopped 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: data_pools [3] 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: metadata_pool 2 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: inline_data disabled 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: balancer 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: bal_rank_mask -1 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: standby_count_wanted 1 2024-09-13T05:32:59.520 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: qdb_cluster leader: 0 members: 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: [mds.foofs.smithi053.rzvrng{0:14506} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.53:6834/2495430984,v1:172.21.15.53:6835/2495430984] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: Standby daemons: 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout: [mds.foofs.smithi110.akbygi{-1:24313} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 ceph-mon[49387]: mon.smithi110@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 bash[49368]: 7c6115f393f971d7a7082dc8e15e6dbb19bf1583a3a06930882431c2e32f0321 2024-09-13T05:32:59.521 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:32:59 smithi110 systemd[1]: Started Ceph mon.smithi110 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:33:00.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 145257 9.95 MB/sec execute 234 sec latency 96.549 ms 2024-09-13T05:33:01.044 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:01.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:01.434 INFO:teuthology.orchestra.run.smithi053.stdout: 5 145584 9.95 MB/sec execute 235 sec latency 73.402 ms 2024-09-13T05:33:02.061 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi053"} : dispatch 2024-09-13T05:33:02.061 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi110"} : dispatch 2024-09-13T05:33:02.061 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: mon.smithi110 calling monitor election 2024-09-13T05:33:02.061 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: mon.smithi053 calling monitor election 2024-09-13T05:33:02.061 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: mon.smithi053 is new leader, mons smithi053,smithi110 in quorum (ranks 0,1) 2024-09-13T05:33:02.061 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: monmap epoch 3 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: last_changed 2024-09-13T05:33:00.861872+0000 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: created 2024-09-13T05:19:06.284452+0000 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: min_mon_release 19 (squid) 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: election_strategy: 1 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: 0: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.smithi053 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: 1: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.smithi110 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: mgrmap e42: smithi053.ygzjut(active, since 18s), standbys: smithi110.rivldx 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: overall HEALTH_OK 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:02.062 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:33:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi053"} : dispatch 2024-09-13T05:33:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mon metadata", "id": "smithi110"} : dispatch 2024-09-13T05:33:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: mon.smithi110 calling monitor election 2024-09-13T05:33:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: mon.smithi053 calling monitor election 2024-09-13T05:33:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: mon.smithi053 is new leader, mons smithi053,smithi110 in quorum (ranks 0,1) 2024-09-13T05:33:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: monmap epoch 3 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: fsid 9f95c28e-718f-11ef-bceb-c7b262605968 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: last_changed 2024-09-13T05:33:00.861872+0000 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: created 2024-09-13T05:19:06.284452+0000 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: min_mon_release 19 (squid) 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: election_strategy: 1 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: 0: [v2:172.21.15.53:3300/0,v1:172.21.15.53:6789/0] mon.smithi053 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: 1: [v2:172.21.15.110:3300/0,v1:172.21.15.110:6789/0] mon.smithi110 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: mgrmap e42: smithi053.ygzjut(active, since 18s), standbys: smithi110.rivldx 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: overall HEALTH_OK 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:02.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:33:02.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 145983 9.96 MB/sec execute 236 sec latency 73.806 ms 2024-09-13T05:33:03.085 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:02 smithi053 ceph-mon[64292]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 329 op/s 2024-09-13T05:33:03.203 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:02 smithi110 ceph-mon[49387]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 329 op/s 2024-09-13T05:33:03.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 146196 9.95 MB/sec execute 237 sec latency 88.389 ms 2024-09-13T05:33:04.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 146355 9.94 MB/sec execute 238 sec latency 67.089 ms 2024-09-13T05:33:05.383 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:33:05.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 146480 9.91 MB/sec execute 239 sec latency 109.408 ms 2024-09-13T05:33:05.670 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:05 smithi053 ceph-mon[64292]: pgmap v14: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 522 op/s 2024-09-13T05:33:05.670 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:05.670 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:05.670 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:05.670 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:05.728 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:05 smithi110 ceph-mon[49387]: pgmap v14: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.3 MiB/s wr, 522 op/s 2024-09-13T05:33:05.728 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:05.728 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:05.728 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:05.728 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:06.293 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:33:06.293 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (2m) 14s ago 12m 19.9M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:33:06.293 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (12m) 14s ago 12m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (11m) 1s ago 11m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (2m) 14s ago 12m 43.1M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (6m) 14s ago 6m 37.0M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (6m) 1s ago 6m 15.9M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (102s) 14s ago 13m 487M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (52s) 1s ago 11m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (34s) 14s ago 13m 33.0M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (7s) 1s ago 11m 26.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (6m) 14s ago 6m 275M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (6m) 1s ago 6m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (2m) 14s ago 12m 10.2M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (111s) 1s ago 11m 9.80M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:33:06.294 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (11m) 14s ago 10m 508M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (10m) 14s ago 10m 429M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (10m) 14s ago 10m 329M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (9m) 14s ago 9m 468M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (9m) 1s ago 9m 487M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (8m) 1s ago 8m 566M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (8m) 1s ago 8m 484M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (8m) 1s ago 8m 357M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:33:06.295 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (59s) 14s ago 11m 46.2M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:33:06.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 146698 9.89 MB/sec execute 240 sec latency 78.445 ms 2024-09-13T05:33:06.700 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:06 smithi053 ceph-mon[64292]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:06.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:06 smithi110 ceph-mon[49387]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:06.855 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:33:06.856 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:33:06.857 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:06.857 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:33:06.857 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T05:33:06.857 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 4 2024-09-13T05:33:06.857 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:33:06.857 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:33:07.368 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "4/23 daemons upgraded", 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:33:07.369 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:33:07.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 146905 9.86 MB/sec execute 241 sec latency 81.047 ms 2024-09-13T05:33:07.475 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:07 smithi053 ceph-mon[64292]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:07.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:07 smithi053 ceph-mon[64292]: pgmap v15: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-09-13T05:33:07.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:07 smithi053 ceph-mon[64292]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:07.476 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:07 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/2459298980' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:07.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:07 smithi110 ceph-mon[49387]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:07.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:07 smithi110 ceph-mon[49387]: pgmap v15: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-09-13T05:33:07.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:07 smithi110 ceph-mon[49387]: from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:07.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:07 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/2459298980' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:07.919 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:33:08.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 147045 9.83 MB/sec execute 242 sec latency 64.639 ms 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/141893316' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T05:33:08.504 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:08.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='client.34148 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:08.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/141893316' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T05:33:08.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:09.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 147239 9.81 MB/sec execute 243 sec latency 108.049 ms 2024-09-13T05:33:09.510 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:09 smithi053 ceph-mon[64292]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-09-13T05:33:09.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:09 smithi053 ceph-mon[64292]: Reconfiguring mon.smithi053 (monmap changed)... 2024-09-13T05:33:09.511 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:09 smithi053 ceph-mon[64292]: Reconfiguring daemon mon.smithi053 on smithi053 2024-09-13T05:33:09.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:09 smithi110 ceph-mon[49387]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-09-13T05:33:09.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:09 smithi110 ceph-mon[49387]: Reconfiguring mon.smithi053 (monmap changed)... 2024-09-13T05:33:09.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:09 smithi110 ceph-mon[49387]: Reconfiguring daemon mon.smithi053 on smithi053 2024-09-13T05:33:10.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 147457 9.81 MB/sec execute 244 sec latency 68.481 ms 2024-09-13T05:33:11.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 147677 9.78 MB/sec execute 245 sec latency 61.850 ms 2024-09-13T05:33:11.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:11 smithi053 ceph-mon[64292]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.8 MiB/s wr, 465 op/s 2024-09-13T05:33:11.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:11.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:11.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T05:33:11.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T05:33:11.592 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:11 smithi110 ceph-mon[49387]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.8 MiB/s wr, 465 op/s 2024-09-13T05:33:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi053.ygzjut", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T05:33:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T05:33:11.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:12.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 147914 9.76 MB/sec execute 246 sec latency 61.239 ms 2024-09-13T05:33:12.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:12 smithi053 ceph-mon[64292]: Reconfiguring mgr.smithi053.ygzjut (monmap changed)... 2024-09-13T05:33:12.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:12 smithi053 ceph-mon[64292]: Reconfiguring daemon mgr.smithi053.ygzjut on smithi053 2024-09-13T05:33:12.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:12 smithi110 ceph-mon[49387]: Reconfiguring mgr.smithi053.ygzjut (monmap changed)... 2024-09-13T05:33:12.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:12 smithi110 ceph-mon[49387]: Reconfiguring daemon mgr.smithi053.ygzjut on smithi053 2024-09-13T05:33:13.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 148161 9.76 MB/sec execute 247 sec latency 101.814 ms 2024-09-13T05:33:13.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:13 smithi053 ceph-mon[64292]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 300 op/s 2024-09-13T05:33:13.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:13.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:13.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:13.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:13.652 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:13 smithi110 ceph-mon[49387]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 300 op/s 2024-09-13T05:33:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:14.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 148384 9.73 MB/sec execute 248 sec latency 56.430 ms 2024-09-13T05:33:14.689 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:14 smithi053 ceph-mon[64292]: Reconfiguring crash.smithi053 (monmap changed)... 2024-09-13T05:33:14.689 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:14 smithi053 ceph-mon[64292]: Reconfiguring daemon crash.smithi053 on smithi053 2024-09-13T05:33:14.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:14 smithi110 ceph-mon[49387]: Reconfiguring crash.smithi053 (monmap changed)... 2024-09-13T05:33:14.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:14 smithi110 ceph-mon[49387]: Reconfiguring daemon crash.smithi053 on smithi053 2024-09-13T05:33:15.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 148569 9.70 MB/sec execute 249 sec latency 94.610 ms 2024-09-13T05:33:15.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:15 smithi053 ceph-mon[64292]: pgmap v19: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-09-13T05:33:15.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:15.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:15.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T05:33:15.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:15 smithi110 ceph-mon[49387]: pgmap v19: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.4 MiB/s wr, 439 op/s 2024-09-13T05:33:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T05:33:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:16.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 148724 9.69 MB/sec execute 250 sec latency 63.472 ms 2024-09-13T05:33:16.730 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:16 smithi053 ceph-mon[64292]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T05:33:16.730 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:16 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.0 on smithi053 2024-09-13T05:33:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:16 smithi110 ceph-mon[49387]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T05:33:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:16 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.0 on smithi053 2024-09-13T05:33:17.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 148982 9.68 MB/sec execute 251 sec latency 53.149 ms 2024-09-13T05:33:17.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:17 smithi053 ceph-mon[64292]: pgmap v20: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.3 MiB/s wr, 245 op/s 2024-09-13T05:33:17.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:17 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:17.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:17 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:17.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:17 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T05:33:17.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:17 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:17 smithi110 ceph-mon[49387]: pgmap v20: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.3 MiB/s wr, 245 op/s 2024-09-13T05:33:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:17 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:17 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:17 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T05:33:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:17 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:18.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 149301 9.69 MB/sec execute 252 sec latency 75.475 ms 2024-09-13T05:33:18.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:18 smithi053 ceph-mon[64292]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T05:33:18.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:18 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.1 on smithi053 2024-09-13T05:33:18.895 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:18 smithi110 ceph-mon[49387]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T05:33:18.895 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:18 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.1 on smithi053 2024-09-13T05:33:19.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 149778 9.70 MB/sec execute 253 sec latency 57.065 ms 2024-09-13T05:33:19.808 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:19 smithi053 ceph-mon[64292]: pgmap v21: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.3 MiB/s wr, 245 op/s 2024-09-13T05:33:19.808 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:19.808 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:19.808 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T05:33:19.808 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:19 smithi110 ceph-mon[49387]: pgmap v21: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.3 MiB/s wr, 245 op/s 2024-09-13T05:33:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T05:33:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:20.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 150196 9.71 MB/sec execute 254 sec latency 65.958 ms 2024-09-13T05:33:20.815 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:20 smithi053 ceph-mon[64292]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T05:33:20.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:20 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.2 on smithi053 2024-09-13T05:33:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:20 smithi110 ceph-mon[49387]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T05:33:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:20 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.2 on smithi053 2024-09-13T05:33:21.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 150561 9.70 MB/sec execute 255 sec latency 55.527 ms 2024-09-13T05:33:21.836 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:21 smithi053 ceph-mon[64292]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 384 op/s 2024-09-13T05:33:21.836 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:21.836 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:21.836 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T05:33:21.836 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:21 smithi110 ceph-mon[49387]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.7 MiB/s wr, 384 op/s 2024-09-13T05:33:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T05:33:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:22.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 150804 9.69 MB/sec execute 256 sec latency 46.416 ms 2024-09-13T05:33:22.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:22 smithi053 ceph-mon[64292]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T05:33:22.620 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:22 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.3 on smithi053 2024-09-13T05:33:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:22 smithi110 ceph-mon[49387]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T05:33:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:22 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.3 on smithi053 2024-09-13T05:33:23.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 151106 9.70 MB/sec execute 257 sec latency 68.801 ms 2024-09-13T05:33:23.644 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:23 smithi053 ceph-mon[64292]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.6 MiB/s wr, 277 op/s 2024-09-13T05:33:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:23 smithi110 ceph-mon[49387]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.6 MiB/s wr, 277 op/s 2024-09-13T05:33:24.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 151403 9.67 MB/sec execute 258 sec latency 60.283 ms 2024-09-13T05:33:24.669 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:24.669 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:24.669 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:24 smithi053 ceph-mon[64292]: Reconfiguring mds.foofs.smithi053.rzvrng (monmap changed)... 2024-09-13T05:33:24.669 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:33:24.669 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:24.669 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:24 smithi053 ceph-mon[64292]: Reconfiguring daemon mds.foofs.smithi053.rzvrng on smithi053 2024-09-13T05:33:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:24 smithi110 ceph-mon[49387]: Reconfiguring mds.foofs.smithi053.rzvrng (monmap changed)... 2024-09-13T05:33:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:33:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:24 smithi110 ceph-mon[49387]: Reconfiguring daemon mds.foofs.smithi053.rzvrng on smithi053 2024-09-13T05:33:25.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 151735 9.67 MB/sec execute 259 sec latency 65.349 ms 2024-09-13T05:33:25.683 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:25 smithi053 ceph-mon[64292]: pgmap v24: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-09-13T05:33:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:25 smithi110 ceph-mon[49387]: pgmap v24: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.9 MiB/s wr, 503 op/s 2024-09-13T05:33:26.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 151921 9.65 MB/sec execute 260 sec latency 66.829 ms 2024-09-13T05:33:27.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 152090 9.62 MB/sec execute 261 sec latency 85.662 ms 2024-09-13T05:33:27.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: pgmap v25: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-09-13T05:33:27.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:27.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:27.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: Reconfiguring crash.smithi110 (monmap changed)... 2024-09-13T05:33:27.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:27.791 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:27.792 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: Reconfiguring daemon crash.smithi110 on smithi110 2024-09-13T05:33:27.792 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: pgmap v25: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-09-13T05:33:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: Reconfiguring crash.smithi110 (monmap changed)... 2024-09-13T05:33:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:27.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: Reconfiguring daemon crash.smithi110 on smithi110 2024-09-13T05:33:27.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:28.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 152272 9.61 MB/sec execute 262 sec latency 75.354 ms 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: pgmap v26: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: Reconfiguring mgr.smithi110.rivldx (monmap changed)... 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:29.348 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:29 smithi053 ceph-mon[64292]: Reconfiguring daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:33:29.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 152673 9.62 MB/sec execute 263 sec latency 54.607 ms 2024-09-13T05:33:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: pgmap v26: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.7 MiB/s wr, 364 op/s 2024-09-13T05:33:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: Reconfiguring mgr.smithi110.rivldx (monmap changed)... 2024-09-13T05:33:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi110.rivldx", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T05:33:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T05:33:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:29.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:29 smithi110 ceph-mon[49387]: Reconfiguring daemon mgr.smithi110.rivldx on smithi110 2024-09-13T05:33:30.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 152892 9.62 MB/sec execute 264 sec latency 80.148 ms 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: Reconfiguring mon.smithi110 (monmap changed)... 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: Reconfiguring daemon mon.smithi110 on smithi110 2024-09-13T05:33:30.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:30 smithi053 ceph-mon[64292]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 514 op/s 2024-09-13T05:33:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: Reconfiguring mon.smithi110 (monmap changed)... 2024-09-13T05:33:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T05:33:30.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T05:33:30.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:30.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: Reconfiguring daemon mon.smithi110 on smithi110 2024-09-13T05:33:30.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:30 smithi110 ceph-mon[49387]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 514 op/s 2024-09-13T05:33:31.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 153094 9.60 MB/sec execute 265 sec latency 71.462 ms 2024-09-13T05:33:32.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:32 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:32.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:32 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:32.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:32 smithi053 ceph-mon[64292]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T05:33:32.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:32 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T05:33:32.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:32 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:32.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:32 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.4 on smithi110 2024-09-13T05:33:32.435 INFO:teuthology.orchestra.run.smithi053.stdout: 5 153265 9.58 MB/sec execute 266 sec latency 73.947 ms 2024-09-13T05:33:32.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:32 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:32.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:32 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:32.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:32 smithi110 ceph-mon[49387]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T05:33:32.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:32 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T05:33:32.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:32 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:32.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:32 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.4 on smithi110 2024-09-13T05:33:33.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 153411 9.57 MB/sec execute 267 sec latency 67.056 ms 2024-09-13T05:33:33.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:33 smithi053 ceph-mon[64292]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-09-13T05:33:33.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:33.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:33.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:33 smithi053 ceph-mon[64292]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T05:33:33.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T05:33:33.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:33.719 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:33 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.5 on smithi110 2024-09-13T05:33:33.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:33 smithi110 ceph-mon[49387]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-09-13T05:33:33.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:33.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:33.987 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:33 smithi110 ceph-mon[49387]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T05:33:33.987 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T05:33:33.987 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:33.987 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:33 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.5 on smithi110 2024-09-13T05:33:34.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 153528 9.55 MB/sec execute 268 sec latency 78.811 ms 2024-09-13T05:33:35.279 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:35 smithi053 ceph-mon[64292]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 514 op/s 2024-09-13T05:33:35.279 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:35.279 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:35.279 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:35 smithi053 ceph-mon[64292]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T05:33:35.279 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T05:33:35.279 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:35.279 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:35 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.6 on smithi110 2024-09-13T05:33:35.409 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:35 smithi110 ceph-mon[49387]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.3 MiB/s wr, 514 op/s 2024-09-13T05:33:35.409 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:35.409 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:35.409 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:35 smithi110 ceph-mon[49387]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T05:33:35.409 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T05:33:35.409 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:35.409 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:35 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.6 on smithi110 2024-09-13T05:33:35.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 153896 9.54 MB/sec execute 269 sec latency 49.882 ms 2024-09-13T05:33:36.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 154169 9.52 MB/sec execute 270 sec latency 54.054 ms 2024-09-13T05:33:36.817 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:36 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:36.817 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:36 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:36.817 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:36 smithi053 ceph-mon[64292]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T05:33:36.817 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:36 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T05:33:36.817 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:36 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:36.817 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:36 smithi053 ceph-mon[64292]: Reconfiguring daemon osd.7 on smithi110 2024-09-13T05:33:36.817 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:36 smithi053 ceph-mon[64292]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.0 MiB/s wr, 288 op/s 2024-09-13T05:33:36.980 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:36 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:36.980 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:36 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:36.980 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:36 smithi110 ceph-mon[49387]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T05:33:36.980 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:36 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T05:33:36.980 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:36 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:36.980 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:36 smithi110 ceph-mon[49387]: Reconfiguring daemon osd.7 on smithi110 2024-09-13T05:33:36.980 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:36 smithi110 ceph-mon[49387]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.0 MiB/s wr, 288 op/s 2024-09-13T05:33:37.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 154395 9.51 MB/sec execute 271 sec latency 78.579 ms 2024-09-13T05:33:38.417 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:33:38.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 154690 9.52 MB/sec execute 272 sec latency 61.910 ms 2024-09-13T05:33:38.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:38.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:38.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: Reconfiguring mds.foofs.smithi110.akbygi (monmap changed)... 2024-09-13T05:33:38.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:33:38.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:38.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: Reconfiguring daemon mds.foofs.smithi110.akbygi on smithi110 2024-09-13T05:33:38.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: pgmap v31: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.0 MiB/s wr, 288 op/s 2024-09-13T05:33:38.866 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:38 smithi053 ceph-mon[64292]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:39.187 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:39.187 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:39.187 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: Reconfiguring mds.foofs.smithi110.akbygi (monmap changed)... 2024-09-13T05:33:39.187 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:33:39.187 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:39.187 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: Reconfiguring daemon mds.foofs.smithi110.akbygi on smithi110 2024-09-13T05:33:39.188 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: pgmap v31: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.0 MiB/s wr, 288 op/s 2024-09-13T05:33:39.188 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:38 smithi110 ceph-mon[49387]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:39.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 155193 9.51 MB/sec execute 273 sec latency 33.143 ms 2024-09-13T05:33:39.580 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:33:39.580 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (3m) 47s ago 13m 19.9M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:33:39.580 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (13m) 47s ago 13m 6400k - 17.2.0 e1d6a67b021e 672c07f748ec 2024-09-13T05:33:39.580 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (12m) 35s ago 12m 6395k - 17.2.0 e1d6a67b021e 3da13b577a57 2024-09-13T05:33:39.580 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (2m) 47s ago 12m 43.1M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:33:39.580 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (7m) 47s ago 7m 37.0M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:33:39.580 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (7m) 35s ago 7m 15.9M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (2m) 47s ago 14m 487M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (85s) 35s ago 12m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (68s) 47s ago 14m 33.0M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (40s) 35s ago 12m 26.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (6m) 47s ago 6m 275M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (6m) 35s ago 6m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (2m) 47s ago 12m 10.2M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (2m) 35s ago 12m 9.80M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (11m) 47s ago 11m 508M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (11m) 47s ago 11m 429M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (10m) 47s ago 10m 329M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (10m) 47s ago 10m 468M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:33:39.581 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (9m) 35s ago 9m 487M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:33:39.582 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (9m) 35s ago 9m 566M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:33:39.582 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (9m) 35s ago 9m 484M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:33:39.582 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (8m) 35s ago 8m 357M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:33:39.582 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (92s) 47s ago 12m 46.2M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:33:40.377 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:40.377 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:40.377 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:40.377 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='client.44119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:40.377 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:33:40.378 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:40.378 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:40.378 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:40.378 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi053"} : dispatch 2024-09-13T05:33:40.378 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi053"}]': finished 2024-09-13T05:33:40.378 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi110"} : dispatch 2024-09-13T05:33:40.378 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:40 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi110"}]': finished 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:40.398 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:33:40.399 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:33:40.399 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:33:40.399 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:33:40.399 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T05:33:40.399 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 4 2024-09-13T05:33:40.399 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:33:40.399 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:33:40.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 155646 9.51 MB/sec execute 274 sec latency 41.708 ms 2024-09-13T05:33:40.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:40.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:40.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:40.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='client.44119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:40.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:33:40.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:40.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:40.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:40.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi053"} : dispatch 2024-09-13T05:33:40.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi053"}]': finished 2024-09-13T05:33:40.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi110"} : dispatch 2024-09-13T05:33:40.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:40 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi110"}]': finished 2024-09-13T05:33:41.023 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:33:41.023 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:33:41.023 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "4/23 daemons upgraded", 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading crash daemons", 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:33:41.024 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:33:41.386 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:41 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all mon 2024-09-13T05:33:41.386 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:41 smithi053 ceph-mon[64292]: pgmap v32: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 444 op/s 2024-09-13T05:33:41.386 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:41 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/178839798' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:41.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 156176 9.55 MB/sec execute 275 sec latency 47.120 ms 2024-09-13T05:33:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:41 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all mon 2024-09-13T05:33:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:41 smithi110 ceph-mon[49387]: pgmap v32: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 444 op/s 2024-09-13T05:33:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:41 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/178839798' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:33:41.750 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:33:42.345 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:42 smithi053 ceph-mon[64292]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:42.345 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:42 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/4052493066' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:33:42.345 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:42.345 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:42.345 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:42.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 156741 9.57 MB/sec execute 276 sec latency 64.384 ms 2024-09-13T05:33:42.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:42 smithi110 ceph-mon[49387]: from='client.34172 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:33:42.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:42 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/4052493066' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:33:42.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:42.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi053", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:42.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:43.354 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:43 smithi053 ceph-mon[64292]: pgmap v33: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.1 MiB/s wr, 294 op/s 2024-09-13T05:33:43.354 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:43 smithi053 ceph-mon[64292]: Upgrade: Updating crash.smithi053 (1/2) 2024-09-13T05:33:43.354 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:43 smithi053 ceph-mon[64292]: Deploying daemon crash.smithi053 on smithi053 2024-09-13T05:33:43.354 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:43.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 157117 9.58 MB/sec execute 277 sec latency 47.519 ms 2024-09-13T05:33:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:43 smithi110 ceph-mon[49387]: pgmap v33: 97 pgs: 97 active+clean; 342 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.1 MiB/s wr, 294 op/s 2024-09-13T05:33:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:43 smithi110 ceph-mon[49387]: Upgrade: Updating crash.smithi053 (1/2) 2024-09-13T05:33:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:43 smithi110 ceph-mon[49387]: Deploying daemon crash.smithi053 on smithi053 2024-09-13T05:33:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:44.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 157503 9.57 MB/sec execute 278 sec latency 47.801 ms 2024-09-13T05:33:45.386 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:45 smithi053 ceph-mon[64292]: pgmap v34: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-09-13T05:33:45.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 157840 9.57 MB/sec execute 279 sec latency 36.275 ms 2024-09-13T05:33:45.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:45 smithi110 ceph-mon[49387]: pgmap v34: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.0 MiB/s wr, 572 op/s 2024-09-13T05:33:46.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 158154 9.57 MB/sec execute 280 sec latency 38.174 ms 2024-09-13T05:33:47.402 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:47 smithi053 ceph-mon[64292]: pgmap v35: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-13T05:33:47.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 158527 9.56 MB/sec execute 281 sec latency 43.610 ms 2024-09-13T05:33:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:47 smithi110 ceph-mon[49387]: pgmap v35: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-13T05:33:48.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 158904 9.57 MB/sec execute 282 sec latency 83.449 ms 2024-09-13T05:33:49.199 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:49.199 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:49.199 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:48 smithi053 ceph-mon[64292]: pgmap v36: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-13T05:33:49.215 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:48 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:49.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:48 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:49.216 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:48 smithi110 ceph-mon[49387]: pgmap v36: 97 pgs: 97 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-13T05:33:49.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 159142 9.55 MB/sec execute 283 sec latency 53.361 ms 2024-09-13T05:33:50.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 159398 9.54 MB/sec execute 284 sec latency 48.937 ms 2024-09-13T05:33:50.742 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:50 smithi053 ceph-mon[64292]: Upgrade: Updating crash.smithi110 (2/2) 2024-09-13T05:33:50.742 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:50 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:50.742 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:50 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:50.742 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:50 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:50.742 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:50 smithi053 ceph-mon[64292]: Deploying daemon crash.smithi110 on smithi110 2024-09-13T05:33:50.915 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:50 smithi110 ceph-mon[49387]: Upgrade: Updating crash.smithi110 (2/2) 2024-09-13T05:33:50.916 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:50 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:50.916 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:50 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi110", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T05:33:50.916 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:50 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:33:50.916 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:50 smithi110 ceph-mon[49387]: Deploying daemon crash.smithi110 on smithi110 2024-09-13T05:33:51.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 159759 9.55 MB/sec execute 285 sec latency 62.730 ms 2024-09-13T05:33:51.546 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:51 smithi053 ceph-mon[64292]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 657 op/s 2024-09-13T05:33:51.839 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:51 smithi110 ceph-mon[49387]: pgmap v37: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.5 MiB/s wr, 657 op/s 2024-09-13T05:33:52.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 160043 9.55 MB/sec execute 286 sec latency 105.561 ms 2024-09-13T05:33:53.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 160434 9.55 MB/sec execute 287 sec latency 66.278 ms 2024-09-13T05:33:53.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:53 smithi053 ceph-mon[64292]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-09-13T05:33:53.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:53.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:53.611 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:33:53.855 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:53 smithi110 ceph-mon[49387]: pgmap v38: 97 pgs: 97 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 501 op/s 2024-09-13T05:33:53.855 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:53.856 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:53.856 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:33:54.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 161138 9.58 MB/sec execute 288 sec latency 23.774 ms 2024-09-13T05:33:55.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 161731 9.61 MB/sec execute 289 sec latency 43.944 ms 2024-09-13T05:33:55.623 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:55 smithi053 ceph-mon[64292]: pgmap v39: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.9 MiB/s wr, 684 op/s 2024-09-13T05:33:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:55 smithi110 ceph-mon[49387]: pgmap v39: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.9 MiB/s wr, 684 op/s 2024-09-13T05:33:56.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 162220 9.61 MB/sec execute 290 sec latency 59.284 ms 2024-09-13T05:33:57.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 162633 9.61 MB/sec execute 291 sec latency 69.717 ms 2024-09-13T05:33:57.673 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:57 smithi053 ceph-mon[64292]: pgmap v40: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-13T05:33:57.673 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:57.673 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:57 smithi110 ceph-mon[49387]: pgmap v40: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-13T05:33:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:58.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 162957 9.62 MB/sec execute 292 sec latency 65.886 ms 2024-09-13T05:33:58.712 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:58.712 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:58.712 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:33:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:33:59.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 163198 9.61 MB/sec execute 293 sec latency 101.548 ms 2024-09-13T05:33:59.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:33:59 smithi053 ceph-mon[64292]: pgmap v41: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-13T05:33:59.912 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:33:59 smithi110 ceph-mon[49387]: pgmap v41: 97 pgs: 97 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-13T05:34:00.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 163472 9.61 MB/sec execute 294 sec latency 100.685 ms 2024-09-13T05:34:01.436 INFO:teuthology.orchestra.run.smithi053.stdout: 5 163788 9.60 MB/sec execute 295 sec latency 55.595 ms 2024-09-13T05:34:01.767 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:01 smithi053 ceph-mon[64292]: pgmap v42: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 712 op/s 2024-09-13T05:34:01.767 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:01.767 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:01 smithi110 ceph-mon[49387]: pgmap v42: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 712 op/s 2024-09-13T05:34:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 163992 9.59 MB/sec execute 296 sec latency 58.428 ms 2024-09-13T05:34:02.811 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.811 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.811 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.811 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:03.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 164228 9.58 MB/sec execute 297 sec latency 76.606 ms 2024-09-13T05:34:03.818 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:03 smithi053 ceph-mon[64292]: pgmap v43: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-13T05:34:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:03 smithi110 ceph-mon[49387]: pgmap v43: 97 pgs: 97 active+clean; 352 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-13T05:34:04.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 164500 9.57 MB/sec execute 298 sec latency 56.847 ms 2024-09-13T05:34:05.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 164806 9.55 MB/sec execute 299 sec latency 59.634 ms 2024-09-13T05:34:05.842 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:05 smithi053 ceph-mon[64292]: pgmap v44: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 649 op/s 2024-09-13T05:34:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:05 smithi110 ceph-mon[49387]: pgmap v44: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 649 op/s 2024-09-13T05:34:06.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 165021 9.54 MB/sec execute 300 sec latency 56.006 ms 2024-09-13T05:34:07.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 165274 9.54 MB/sec execute 301 sec latency 46.583 ms 2024-09-13T05:34:07.915 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: pgmap v45: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 466 op/s 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi053"} : dispatch 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi053"}]': finished 2024-09-13T05:34:07.916 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi110"} : dispatch 2024-09-13T05:34:07.917 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi110"}]': finished 2024-09-13T05:34:07.917 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T05:34:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: pgmap v45: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 466 op/s 2024-09-13T05:34:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:34:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:34:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi053"} : dispatch 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi053"}]': finished 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi110"} : dispatch 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi110"}]': finished 2024-09-13T05:34:07.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T05:34:08.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 165594 9.53 MB/sec execute 302 sec latency 50.060 ms 2024-09-13T05:34:08.675 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:08 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all crash 2024-09-13T05:34:08.675 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:08 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T05:34:08.675 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:08 smithi053 ceph-mon[64292]: Upgrade: osd.0 is safe to restart 2024-09-13T05:34:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:08 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all crash 2024-09-13T05:34:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:08 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T05:34:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:08 smithi110 ceph-mon[49387]: Upgrade: osd.0 is safe to restart 2024-09-13T05:34:09.437 INFO:teuthology.orchestra.run.smithi053.stdout: 5 165901 9.52 MB/sec execute 303 sec latency 61.089 ms 2024-09-13T05:34:09.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:09 smithi053 ceph-mon[64292]: pgmap v46: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 466 op/s 2024-09-13T05:34:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:09 smithi110 ceph-mon[49387]: pgmap v46: 97 pgs: 97 active+clean; 350 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 466 op/s 2024-09-13T05:34:10.445 INFO:teuthology.orchestra.run.smithi053.stdout: 5 166157 9.51 MB/sec execute 304 sec latency 69.952 ms 2024-09-13T05:34:11.291 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:11 smithi053 ceph-mon[64292]: pgmap v47: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 637 op/s 2024-09-13T05:34:11.291 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:11 smithi053 ceph-mon[64292]: Upgrade: Updating osd.0 2024-09-13T05:34:11.291 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:11.291 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T05:34:11.291 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:11 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:34:11.291 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:11 smithi053 ceph-mon[64292]: Deploying daemon osd.0 on smithi053 2024-09-13T05:34:11.445 INFO:teuthology.orchestra.run.smithi053.stdout: 5 166452 9.52 MB/sec execute 305 sec latency 52.101 ms 2024-09-13T05:34:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:11 smithi110 ceph-mon[49387]: pgmap v47: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 637 op/s 2024-09-13T05:34:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:11 smithi110 ceph-mon[49387]: Upgrade: Updating osd.0 2024-09-13T05:34:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T05:34:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:11 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:34:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:11 smithi110 ceph-mon[49387]: Deploying daemon osd.0 on smithi053 2024-09-13T05:34:12.243 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:34:12.447 INFO:teuthology.orchestra.run.smithi053.stdout: 5 166642 9.50 MB/sec execute 306 sec latency 67.863 ms 2024-09-13T05:34:13.317 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:13 smithi053 ceph-mon[64292]: pgmap v48: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 330 op/s 2024-09-13T05:34:13.317 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:13 smithi053 ceph-mon[64292]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:13.317 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:13.386 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:34:13.386 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (3m) 12s ago 13m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:34:13.386 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (25s) 12s ago 13m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:34:13.386 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (20s) 16s ago 13m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:34:13.386 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (3m) 12s ago 13m 43.0M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (7m) 12s ago 7m 37.6M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (7m) 16s ago 7m 16.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (2m) 12s ago 14m 509M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (119s) 16s ago 12m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (101s) 12s ago 15m 43.7M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (74s) 16s ago 12m 36.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (7m) 12s ago 7m 276M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (7m) 16s ago 7m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (3m) 12s ago 13m 9.83M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (2m) 16s ago 12m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (12m) 12s ago 12m 331M 1937M 17.2.0 e1d6a67b021e 3615e4d53f48 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (11m) 12s ago 11m 482M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:34:13.387 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (11m) 12s ago 11m 384M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:34:13.388 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (10m) 12s ago 10m 516M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:34:13.388 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (10m) 16s ago 10m 527M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:34:13.388 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (9m) 16s ago 9m 337M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:34:13.388 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (9m) 16s ago 9m 525M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:34:13.388 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (9m) 16s ago 9m 431M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:34:13.388 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (2m) 12s ago 12m 48.6M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:34:13.456 INFO:teuthology.orchestra.run.smithi053.stdout: 5 166969 9.51 MB/sec execute 307 sec latency 61.968 ms 2024-09-13T05:34:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:13 smithi110 ceph-mon[49387]: pgmap v48: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.4 MiB/s wr, 330 op/s 2024-09-13T05:34:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:13 smithi110 ceph-mon[49387]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:13.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:14.159 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:34:14.159 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:34:14.159 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:34:14.159 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 4 2024-09-13T05:34:14.160 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:34:14.161 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:34:14.336 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:14 smithi053 ceph-mon[64292]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:14.336 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:14 smithi053 ceph-mon[64292]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:14.456 INFO:teuthology.orchestra.run.smithi053.stdout: 5 167313 9.51 MB/sec execute 308 sec latency 44.565 ms 2024-09-13T05:34:14.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:14 smithi110 ceph-mon[49387]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:14.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:14 smithi110 ceph-mon[49387]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:14.700 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:34:14.700 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:34:14.700 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:34:14.700 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:34:14.700 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "6/23 daemons upgraded", 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:34:14.701 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:34:15.346 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:15 smithi053 ceph-mon[64292]: pgmap v49: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 493 op/s 2024-09-13T05:34:15.346 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:15 smithi053 ceph-mon[64292]: mgrmap e43: smithi053.ygzjut(active, since 92s), standbys: smithi110.rivldx 2024-09-13T05:34:15.346 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:15 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3192827916' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:15.346 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:15 smithi053 systemd[1]: Stopping Ceph osd.0 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:34:15.442 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:34:15.456 INFO:teuthology.orchestra.run.smithi053.stdout: 5 167535 9.50 MB/sec execute 309 sec latency 48.492 ms 2024-09-13T05:34:15.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:15 smithi110 ceph-mon[49387]: pgmap v49: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 493 op/s 2024-09-13T05:34:15.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:15 smithi110 ceph-mon[49387]: mgrmap e43: smithi053.ygzjut(active, since 92s), standbys: smithi110.rivldx 2024-09-13T05:34:15.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:15 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3192827916' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:15.606 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:15 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[34214]: 2024-09-13T05:34:15.585+0000 7f9255669700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:34:15.606 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:15 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[34214]: 2024-09-13T05:34:15.585+0000 7f9255669700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-13T05:34:15.606 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:15 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[34214]: 2024-09-13T05:34:15.585+0000 7f9255669700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:34:16.139 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:16 smithi053 ceph-mon[64292]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:16.139 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:16 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3237710914' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:34:16.139 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:16 smithi053 ceph-mon[64292]: osd.0 marked itself down and dead 2024-09-13T05:34:16.456 INFO:teuthology.orchestra.run.smithi053.stdout: 5 167722 9.48 MB/sec execute 310 sec latency 298.553 ms 2024-09-13T05:34:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:16 smithi110 ceph-mon[49387]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:16 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3237710914' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:34:16.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:16 smithi110 ceph-mon[49387]: osd.0 marked itself down and dead 2024-09-13T05:34:17.160 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:16 smithi053 podman[68725]: 2024-09-13 05:34:16.814175309 +0000 UTC m=+1.365099915 container died 3615e4d53f48610e4dffec1dc535524d7af89e02a69fd0718198d0904e7527df (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.openshift.expose-services=, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, version=8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, distribution-scope=public, com.redhat.component=centos-stream-container) 2024-09-13T05:34:17.416 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:17 smithi053 ceph-mon[64292]: pgmap v50: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 332 op/s 2024-09-13T05:34:17.416 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:17 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:34:17.416 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:17 smithi053 ceph-mon[64292]: osdmap e73: 8 total, 7 up, 8 in 2024-09-13T05:34:17.416 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:17 smithi053 podman[68725]: 2024-09-13 05:34:17.393021151 +0000 UTC m=+1.943945751 container cleanup 3615e4d53f48610e4dffec1dc535524d7af89e02a69fd0718198d0904e7527df (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, name=centos-stream, version=8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-type=git, distribution-scope=public, release=754, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream) 2024-09-13T05:34:17.416 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:17 smithi053 bash[68725]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0 2024-09-13T05:34:17.461 INFO:teuthology.orchestra.run.smithi053.stdout: 5 167826 9.45 MB/sec execute 311 sec latency 1014.594 ms 2024-09-13T05:34:17.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:17 smithi110 ceph-mon[49387]: pgmap v50: 97 pgs: 97 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 332 op/s 2024-09-13T05:34:17.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:17 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:34:17.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:17 smithi110 ceph-mon[49387]: osdmap e73: 8 total, 7 up, 8 in 2024-09-13T05:34:17.918 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:17 smithi053 podman[68742]: 2024-09-13 05:34:17.735324123 +0000 UTC m=+0.923411702 container remove 3615e4d53f48610e4dffec1dc535524d7af89e02a69fd0718198d0904e7527df (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, distribution-scope=public, name=centos-stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True) 2024-09-13T05:34:18.173 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:18 smithi053 ceph-mon[64292]: osdmap e74: 8 total, 7 up, 8 in 2024-09-13T05:34:18.445 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:18 smithi053 podman[68829]: 2024-09-13 05:34:18.213675939 +0000 UTC m=+0.028319761 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:34:18.445 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:18 smithi053 podman[68829]: 2024-09-13 05:34:18.339012874 +0000 UTC m=+0.153656692 container create 5629fd059bb4982976129425adc1f5f99a9fee6d48748647003e0ae7ae45f1eb (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:34:18.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 168088 9.45 MB/sec execute 312 sec latency 61.853 ms 2024-09-13T05:34:18.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:18 smithi110 ceph-mon[49387]: osdmap e74: 8 total, 7 up, 8 in 2024-09-13T05:34:18.697 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:18 smithi053 podman[68829]: 2024-09-13 05:34:18.625565158 +0000 UTC m=+0.440208986 container init 5629fd059bb4982976129425adc1f5f99a9fee6d48748647003e0ae7ae45f1eb (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T05:34:18.697 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:18 smithi053 podman[68829]: 2024-09-13 05:34:18.639230829 +0000 UTC m=+0.453874657 container start 5629fd059bb4982976129425adc1f5f99a9fee6d48748647003e0ae7ae45f1eb (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-13T05:34:18.948 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:18 smithi053 podman[68829]: 2024-09-13 05:34:18.77645818 +0000 UTC m=+0.591102231 container attach 5629fd059bb4982976129425adc1f5f99a9fee6d48748647003e0ae7ae45f1eb (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:34:19.205 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:19 smithi053 ceph-mon[64292]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.6 MiB/s wr, 244 op/s 2024-09-13T05:34:19.205 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:19 smithi053 podman[68829]: 2024-09-13 05:34:19.024000471 +0000 UTC m=+0.838644298 container died 5629fd059bb4982976129425adc1f5f99a9fee6d48748647003e0ae7ae45f1eb (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T05:34:19.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 168544 9.45 MB/sec execute 313 sec latency 41.082 ms 2024-09-13T05:34:19.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:19 smithi110 ceph-mon[49387]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.6 MiB/s wr, 244 op/s 2024-09-13T05:34:20.228 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:19 smithi053 podman[68852]: 2024-09-13 05:34:19.97559233 +0000 UTC m=+0.940095321 container remove 5629fd059bb4982976129425adc1f5f99a9fee6d48748647003e0ae7ae45f1eb (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:34:20.228 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:20 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T05:34:20.228 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:20 smithi053 systemd[1]: Stopped Ceph osd.0 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:34:20.228 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:20 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 58.121s CPU time. 2024-09-13T05:34:20.229 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:20 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 220/1635 objects degraded (13.456%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:20.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 168958 9.45 MB/sec execute 314 sec latency 54.046 ms 2024-09-13T05:34:20.481 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:20 smithi053 systemd[1]: Starting Ceph osd.0 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:34:20.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:20 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 220/1635 objects degraded (13.456%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:20.747 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:20 smithi053 podman[68946]: 2024-09-13 05:34:20.59837656 +0000 UTC m=+0.029523854 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:34:20.747 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:20 smithi053 podman[68946]: 2024-09-13 05:34:20.727442141 +0000 UTC m=+0.158589429 container create ad87b856634b97535ec78af637715561028f1604ddedb1fa3e18a92ff8a3dad3 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-13T05:34:21.259 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:21 smithi053 ceph-mon[64292]: pgmap v54: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 458 op/s; 220/1635 objects degraded (13.456%) 2024-09-13T05:34:21.259 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:21 smithi053 podman[68946]: 2024-09-13 05:34:21.147860947 +0000 UTC m=+0.579008248 container init ad87b856634b97535ec78af637715561028f1604ddedb1fa3e18a92ff8a3dad3 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T05:34:21.259 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:21 smithi053 podman[68946]: 2024-09-13 05:34:21.163932927 +0000 UTC m=+0.595080221 container start ad87b856634b97535ec78af637715561028f1604ddedb1fa3e18a92ff8a3dad3 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-13T05:34:21.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 169434 9.46 MB/sec execute 315 sec latency 32.386 ms 2024-09-13T05:34:21.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:21 smithi110 ceph-mon[49387]: pgmap v54: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 458 op/s; 220/1635 objects degraded (13.456%) 2024-09-13T05:34:21.517 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:21 smithi053 podman[68946]: 2024-09-13 05:34:21.340961844 +0000 UTC m=+0.772109136 container attach ad87b856634b97535ec78af637715561028f1604ddedb1fa3e18a92ff8a3dad3 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T05:34:21.517 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:21 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:34:21.517 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:21 smithi053 bash[68946]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:34:21.517 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:21 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:34:21.517 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:21 smithi053 bash[68946]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:34:22.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 169738 9.45 MB/sec execute 316 sec latency 44.995 ms 2024-09-13T05:34:23.301 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:23 smithi053 ceph-mon[64292]: pgmap v55: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.1 MiB/s wr, 214 op/s; 220/1635 objects degraded (13.456%) 2024-09-13T05:34:23.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 170052 9.46 MB/sec execute 317 sec latency 41.788 ms 2024-09-13T05:34:23.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:23 smithi110 ceph-mon[49387]: pgmap v55: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.1 MiB/s wr, 214 op/s; 220/1635 objects degraded (13.456%) 2024-09-13T05:34:24.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 170377 9.46 MB/sec execute 318 sec latency 42.623 ms 2024-09-13T05:34:24.583 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T05:34:24.583 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 bash[68946]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T05:34:24.583 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T05:34:24.583 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 bash[68946]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T05:34:25.104 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T05:34:25.104 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 bash[68946]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T05:34:25.104 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 bash[68946]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 bash[68946]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 bash[68946]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate[68960]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 bash[68946]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T05:34:25.105 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:24 smithi053 podman[68946]: 2024-09-13 05:34:24.899093359 +0000 UTC m=+4.330240653 container died ad87b856634b97535ec78af637715561028f1604ddedb1fa3e18a92ff8a3dad3 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T05:34:25.355 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:25 smithi053 ceph-mon[64292]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 564 op/s; 227/1665 objects degraded (13.634%) 2024-09-13T05:34:25.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 170737 9.47 MB/sec execute 319 sec latency 93.946 ms 2024-09-13T05:34:25.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:25 smithi110 ceph-mon[49387]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.4 MiB/s wr, 564 op/s; 227/1665 objects degraded (13.634%) 2024-09-13T05:34:25.868 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:25 smithi053 podman[69134]: 2024-09-13 05:34:25.821328965 +0000 UTC m=+0.913276703 container remove ad87b856634b97535ec78af637715561028f1604ddedb1fa3e18a92ff8a3dad3 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T05:34:26.119 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:26 smithi053 podman[69190]: 2024-09-13 05:34:25.966934347 +0000 UTC m=+0.021009711 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:34:26.119 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:26 smithi053 podman[69190]: 2024-09-13 05:34:26.080989364 +0000 UTC m=+0.135064720 container create a83746e4a39d264962a69b005c43599c1ce5552fa6b1a6e554254a8827aa11ce (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T05:34:26.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 171054 9.46 MB/sec execute 320 sec latency 63.543 ms 2024-09-13T05:34:26.624 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:26 smithi053 podman[69190]: 2024-09-13 05:34:26.411496044 +0000 UTC m=+0.465571417 container init a83746e4a39d264962a69b005c43599c1ce5552fa6b1a6e554254a8827aa11ce (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:34:26.624 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:26 smithi053 podman[69190]: 2024-09-13 05:34:26.418972714 +0000 UTC m=+0.473048076 container start a83746e4a39d264962a69b005c43599c1ce5552fa6b1a6e554254a8827aa11ce (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-13T05:34:26.624 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:26 smithi053 ceph-osd[69209]: -- 172.21.15.53:0/3128894637 <== mon.0 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5590227161e0 con 0x5590218b1c00 2024-09-13T05:34:26.624 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:26 smithi053 bash[69190]: a83746e4a39d264962a69b005c43599c1ce5552fa6b1a6e554254a8827aa11ce 2024-09-13T05:34:26.624 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:26 smithi053 systemd[1]: Started Ceph osd.0 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:34:27.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 171372 9.46 MB/sec execute 321 sec latency 57.172 ms 2024-09-13T05:34:27.913 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-mon[64292]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 457 op/s; 227/1665 objects degraded (13.634%) 2024-09-13T05:34:27.913 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:27.913 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:27.913 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:34:27.913 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 227/1665 objects degraded (13.634%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:27.913 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:27.914 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:27 smithi110 ceph-mon[49387]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.4 MiB/s wr, 457 op/s; 227/1665 objects degraded (13.634%) 2024-09-13T05:34:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:34:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:27 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 227/1665 objects degraded (13.634%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:27.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:28.164 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:27 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[69205]: 2024-09-13T05:34:27.967+0000 7ff40791a640 -1 Falling back to public interface 2024-09-13T05:34:28.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 171656 9.45 MB/sec execute 322 sec latency 67.595 ms 2024-09-13T05:34:29.466 INFO:teuthology.orchestra.run.smithi053.stdout: 5 172031 9.44 MB/sec execute 323 sec latency 69.248 ms 2024-09-13T05:34:29.940 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:29 smithi053 ceph-mon[64292]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 415 op/s; 227/1665 objects degraded (13.634%) 2024-09-13T05:34:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:29 smithi110 ceph-mon[49387]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 350 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 415 op/s; 227/1665 objects degraded (13.634%) 2024-09-13T05:34:30.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 172427 9.45 MB/sec execute 324 sec latency 66.010 ms 2024-09-13T05:34:31.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 172891 9.46 MB/sec execute 325 sec latency 52.277 ms 2024-09-13T05:34:31.695 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:31 smithi053 ceph-mon[64292]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 224/1662 objects degraded (13.478%) 2024-09-13T05:34:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:31 smithi110 ceph-mon[49387]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 224/1662 objects degraded (13.478%) 2024-09-13T05:34:32.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 173139 9.44 MB/sec execute 326 sec latency 57.245 ms 2024-09-13T05:34:32.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:32 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 224/1662 objects degraded (13.478%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:32 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 224/1662 objects degraded (13.478%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:33.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 173400 9.43 MB/sec execute 327 sec latency 74.734 ms 2024-09-13T05:34:33.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:33 smithi053 ceph-mon[64292]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 428 op/s; 224/1662 objects degraded (13.478%) 2024-09-13T05:34:33.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:33 smithi110 ceph-mon[49387]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 354 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 428 op/s; 224/1662 objects degraded (13.478%) 2024-09-13T05:34:34.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 173627 9.44 MB/sec execute 328 sec latency 57.500 ms 2024-09-13T05:34:34.572 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:34 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[69205]: 2024-09-13T05:34:34.401+0000 7ff40791a640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-13T05:34:35.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 173895 9.44 MB/sec execute 329 sec latency 80.806 ms 2024-09-13T05:34:35.599 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:35 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[69205]: 2024-09-13T05:34:35.488+0000 7ff40791a640 -1 osd.0 72 log_to_monitors true 2024-09-13T05:34:35.850 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:35 smithi053 ceph-mon[64292]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.4 MiB/s wr, 644 op/s; 213/1629 objects degraded (13.076%) 2024-09-13T05:34:35.850 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:35 smithi053 ceph-mon[64292]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T05:34:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:35 smithi110 ceph-mon[49387]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.4 MiB/s wr, 644 op/s; 213/1629 objects degraded (13.076%) 2024-09-13T05:34:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:35 smithi110 ceph-mon[49387]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T05:34:36.101 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[69205]: 2024-09-13T05:34:36.063+0000 7ff3fb6bc640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:34:36.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 174290 9.44 MB/sec execute 330 sec latency 56.968 ms 2024-09-13T05:34:36.863 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-mon[64292]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T05:34:36.863 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-mon[64292]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T05:34:36.863 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-mon[64292]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:34:36.863 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-mon[64292]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 213/1629 objects degraded (13.076%) 2024-09-13T05:34:36.863 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-mon[64292]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' 2024-09-13T05:34:36.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:36.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:36 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:36 smithi110 ceph-mon[49387]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T05:34:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:36 smithi110 ceph-mon[49387]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T05:34:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:36 smithi110 ceph-mon[49387]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:34:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:36 smithi110 ceph-mon[49387]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 493 op/s; 213/1629 objects degraded (13.076%) 2024-09-13T05:34:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:36 smithi110 ceph-mon[49387]: from='osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559]' entity='osd.0' 2024-09-13T05:34:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:36 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:36 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:37.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 174466 9.43 MB/sec execute 331 sec latency 434.225 ms 2024-09-13T05:34:37.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:37.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:37.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:34:37.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559] boot 2024-09-13T05:34:37.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T05:34:37.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T05:34:37.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 213/1629 objects degraded (13.076%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:37.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:37 smithi053 ceph-mon[64292]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T05:34:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:34:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: osd.0 [v2:172.21.15.53:6802/33198559,v1:172.21.15.53:6803/33198559] boot 2024-09-13T05:34:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: osdmap e76: 8 total, 8 up, 8 in 2024-09-13T05:34:38.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T05:34:38.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 213/1629 objects degraded (13.076%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:38.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:37 smithi110 ceph-mon[49387]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T05:34:38.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 174466 9.40 MB/sec execute 332 sec latency 1434.264 ms 2024-09-13T05:34:39.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 174649 9.39 MB/sec execute 333 sec latency 1792.979 ms 2024-09-13T05:34:39.520 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:39 smithi053 ceph-mon[64292]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 432 op/s; 213/1629 objects degraded (13.076%) 2024-09-13T05:34:39.520 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:39 smithi053 ceph-mon[64292]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T05:34:39.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:39 smithi110 ceph-mon[49387]: pgmap v66: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 432 op/s; 213/1629 objects degraded (13.076%) 2024-09-13T05:34:39.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:39 smithi110 ceph-mon[49387]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T05:34:40.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 174746 9.36 MB/sec execute 334 sec latency 591.255 ms 2024-09-13T05:34:41.064 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:40 smithi053 ceph-mon[64292]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T05:34:41.064 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:40 smithi053 ceph-mon[64292]: pgmap v69: 97 pgs: 13 remapped+peering, 5 active+undersized, 3 active+undersized+degraded, 76 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.9 MiB/s wr, 414 op/s; 20/1677 objects degraded (1.193%); 22 B/s, 4 objects/s recovering 2024-09-13T05:34:41.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:40 smithi110 ceph-mon[49387]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T05:34:41.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:40 smithi110 ceph-mon[49387]: pgmap v69: 97 pgs: 13 remapped+peering, 5 active+undersized, 3 active+undersized+degraded, 76 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.9 MiB/s wr, 414 op/s; 20/1677 objects degraded (1.193%); 22 B/s, 4 objects/s recovering 2024-09-13T05:34:41.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 174882 9.35 MB/sec execute 335 sec latency 1037.609 ms 2024-09-13T05:34:42.078 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:41 smithi053 ceph-mon[64292]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T05:34:42.078 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.078 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.078 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:34:42.078 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:34:42.078 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.078 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:41 smithi110 ceph-mon[49387]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T05:34:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:34:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:34:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:42.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 175166 9.33 MB/sec execute 336 sec latency 931.489 ms 2024-09-13T05:34:43.089 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T05:34:43.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:34:43.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:43.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:43.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:43.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T05:34:43.090 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T05:34:43.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T05:34:43.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: pgmap v72: 97 pgs: 2 peering, 11 remapped+peering, 84 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 5.1 MiB/s wr, 422 op/s; 22 B/s, 10 objects/s recovering 2024-09-13T05:34:43.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 20/1677 objects degraded (1.193%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:43.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:43.091 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T05:34:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:34:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: pgmap v72: 97 pgs: 2 peering, 11 remapped+peering, 84 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 5.1 MiB/s wr, 422 op/s; 22 B/s, 10 objects/s recovering 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 20/1677 objects degraded (1.193%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:43.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 175490 9.33 MB/sec execute 337 sec latency 90.387 ms 2024-09-13T05:34:44.095 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:43 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1677 objects degraded (1.193%), 3 pgs degraded) 2024-09-13T05:34:44.095 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:43 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:34:44.095 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:43 smithi053 ceph-mon[64292]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T05:34:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:43 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1677 objects degraded (1.193%), 3 pgs degraded) 2024-09-13T05:34:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:43 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:34:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:43 smithi110 ceph-mon[49387]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T05:34:44.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 175788 9.33 MB/sec execute 338 sec latency 509.694 ms 2024-09-13T05:34:45.118 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:44 smithi053 ceph-mon[64292]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T05:34:45.118 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:44 smithi053 ceph-mon[64292]: pgmap v75: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 remapped+peering, 89 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.1 MiB/s wr, 342 op/s; 26/1689 objects degraded (1.539%); 3.6 MiB/s, 36 objects/s recovering 2024-09-13T05:34:45.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:44 smithi110 ceph-mon[49387]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T05:34:45.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:44 smithi110 ceph-mon[49387]: pgmap v75: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 remapped+peering, 89 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.1 MiB/s wr, 342 op/s; 26/1689 objects degraded (1.539%); 3.6 MiB/s, 36 objects/s recovering 2024-09-13T05:34:45.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 176205 9.33 MB/sec execute 339 sec latency 1003.550 ms 2024-09-13T05:34:46.117 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:34:46.157 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:45 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 26/1689 objects degraded (1.539%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:46.157 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:45 smithi053 ceph-mon[64292]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T05:34:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:45 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 26/1689 objects degraded (1.539%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:34:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:45 smithi110 ceph-mon[49387]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T05:34:46.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 176608 9.35 MB/sec execute 340 sec latency 73.033 ms 2024-09-13T05:34:47.193 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:46 smithi053 ceph-mon[64292]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T05:34:47.193 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:46 smithi053 ceph-mon[64292]: pgmap v78: 97 pgs: 4 peering, 93 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.6 MiB/s wr, 372 op/s; 3.6 MiB/s, 57 objects/s recovering 2024-09-13T05:34:47.193 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:46 smithi053 ceph-mon[64292]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:47.193 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:46 smithi053 ceph-mon[64292]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:47.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:46 smithi110 ceph-mon[49387]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T05:34:47.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:46 smithi110 ceph-mon[49387]: pgmap v78: 97 pgs: 4 peering, 93 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.6 MiB/s wr, 372 op/s; 3.6 MiB/s, 57 objects/s recovering 2024-09-13T05:34:47.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:46 smithi110 ceph-mon[49387]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:47.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:46 smithi110 ceph-mon[49387]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:47.332 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:34:47.332 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (4m) 11s ago 14m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:34:47.332 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (59s) 11s ago 14m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:34:47.332 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (54s) 50s ago 13m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:34:47.332 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (4m) 11s ago 13m 43.1M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:34:47.332 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (8m) 11s ago 8m 37.7M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:34:47.332 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (8m) 50s ago 8m 16.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (3m) 11s ago 15m 511M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (2m) 50s ago 13m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (2m) 11s ago 15m 47.3M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (108s) 50s ago 13m 36.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (7m) 11s ago 7m 275M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (7m) 50s ago 7m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (3m) 11s ago 13m 10.4M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (3m) 50s ago 13m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (21s) 11s ago 12m 11.4M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (12m) 11s ago 12m 532M 1937M 17.2.0 e1d6a67b021e d22368ce7251 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (11m) 11s ago 11m 411M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:34:47.333 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (11m) 11s ago 11m 541M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:34:47.334 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (10m) 50s ago 10m 527M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:34:47.334 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (10m) 50s ago 10m 337M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:34:47.334 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (10m) 50s ago 10m 525M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:34:47.334 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (9m) 50s ago 9m 431M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:34:47.334 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (2m) 11s ago 13m 48.7M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:34:47.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 176955 9.34 MB/sec execute 341 sec latency 49.179 ms 2024-09-13T05:34:48.025 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:34:48.026 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 5 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:34:48.027 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:34:48.197 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:47 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1689 objects degraded (1.539%), 2 pgs degraded) 2024-09-13T05:34:48.197 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:47 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:34:48.197 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:47 smithi053 ceph-mon[64292]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:48.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:47 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 26/1689 objects degraded (1.539%), 2 pgs degraded) 2024-09-13T05:34:48.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:47 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:34:48.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:47 smithi110 ceph-mon[49387]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:48.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 177318 9.35 MB/sec execute 342 sec latency 44.628 ms 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:34:48.584 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "7/23 daemons upgraded", 2024-09-13T05:34:48.585 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:34:48.585 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:34:48.585 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:34:49.151 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:34:49.221 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:48 smithi053 ceph-mon[64292]: pgmap v79: 97 pgs: 4 peering, 93 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 292 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-09-13T05:34:49.221 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:48 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/4062531453' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:49.221 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:48 smithi053 ceph-mon[64292]: from='client.44177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:49.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:48 smithi110 ceph-mon[49387]: pgmap v79: 97 pgs: 4 peering, 93 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.8 MiB/s wr, 292 op/s; 2.8 MiB/s, 45 objects/s recovering 2024-09-13T05:34:49.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:48 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/4062531453' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:34:49.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:48 smithi110 ceph-mon[49387]: from='client.44177 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:34:49.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 177549 9.33 MB/sec execute 343 sec latency 49.340 ms 2024-09-13T05:34:50.007 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:49 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/2409809982' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:34:50.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 177885 9.35 MB/sec execute 344 sec latency 44.370 ms 2024-09-13T05:34:50.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:49 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/2409809982' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:34:51.015 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:50 smithi053 ceph-mon[64292]: pgmap v80: 97 pgs: 1 peering, 96 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 414 op/s; 0 B/s, 17 objects/s recovering 2024-09-13T05:34:51.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 178527 9.35 MB/sec execute 345 sec latency 26.587 ms 2024-09-13T05:34:51.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:50 smithi110 ceph-mon[49387]: pgmap v80: 97 pgs: 1 peering, 96 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 414 op/s; 0 B/s, 17 objects/s recovering 2024-09-13T05:34:52.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 179237 9.38 MB/sec execute 346 sec latency 31.473 ms 2024-09-13T05:34:53.146 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:53 smithi053 ceph-mon[64292]: pgmap v81: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.5 MiB/s wr, 331 op/s; 0 B/s, 13 objects/s recovering 2024-09-13T05:34:53.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 179646 9.39 MB/sec execute 347 sec latency 71.369 ms 2024-09-13T05:34:53.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:53 smithi110 ceph-mon[49387]: pgmap v81: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.5 MiB/s wr, 331 op/s; 0 B/s, 13 objects/s recovering 2024-09-13T05:34:54.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 179952 9.38 MB/sec execute 348 sec latency 52.804 ms 2024-09-13T05:34:55.168 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:55 smithi053 ceph-mon[64292]: pgmap v82: 97 pgs: 97 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 643 op/s; 0 B/s, 11 objects/s recovering 2024-09-13T05:34:55.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 180256 9.39 MB/sec execute 349 sec latency 43.688 ms 2024-09-13T05:34:55.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:55 smithi110 ceph-mon[49387]: pgmap v82: 97 pgs: 97 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 6.5 MiB/s wr, 643 op/s; 0 B/s, 11 objects/s recovering 2024-09-13T05:34:56.467 INFO:teuthology.orchestra.run.smithi053.stdout: 5 180498 9.37 MB/sec execute 350 sec latency 61.978 ms 2024-09-13T05:34:57.214 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:57 smithi053 ceph-mon[64292]: pgmap v83: 97 pgs: 97 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 586 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:34:57.214 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T05:34:57.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 180848 9.38 MB/sec execute 351 sec latency 40.255 ms 2024-09-13T05:34:57.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:57 smithi110 ceph-mon[49387]: pgmap v83: 97 pgs: 97 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 586 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:34:57.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T05:34:58.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 181275 9.39 MB/sec execute 352 sec latency 74.575 ms 2024-09-13T05:34:58.741 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:58 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T05:34:58.741 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:58 smithi053 ceph-mon[64292]: Upgrade: osd.1 is safe to restart 2024-09-13T05:34:58.741 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:58.741 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:58.741 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:58.897 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:58 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T05:34:58.898 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:58 smithi110 ceph-mon[49387]: Upgrade: osd.1 is safe to restart 2024-09-13T05:34:58.898 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:58.898 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:34:58.898 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:59.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 181591 9.39 MB/sec execute 353 sec latency 62.083 ms 2024-09-13T05:34:59.764 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:59 smithi053 ceph-mon[64292]: pgmap v84: 97 pgs: 97 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:34:59.764 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:59 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:59.764 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:59 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T05:34:59.764 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:34:59 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:34:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:59 smithi110 ceph-mon[49387]: pgmap v84: 97 pgs: 97 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 492 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:34:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:59 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:34:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:59 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T05:34:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:34:59 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:35:00.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 181993 9.38 MB/sec execute 354 sec latency 60.075 ms 2024-09-13T05:35:00.768 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:00 smithi053 ceph-mon[64292]: Upgrade: Updating osd.1 2024-09-13T05:35:00.768 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:00 smithi053 ceph-mon[64292]: Deploying daemon osd.1 on smithi053 2024-09-13T05:35:00.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:00 smithi110 ceph-mon[49387]: Upgrade: Updating osd.1 2024-09-13T05:35:00.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:00 smithi110 ceph-mon[49387]: Deploying daemon osd.1 on smithi053 2024-09-13T05:35:01.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 182276 9.37 MB/sec execute 355 sec latency 68.232 ms 2024-09-13T05:35:01.792 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:01 smithi053 ceph-mon[64292]: pgmap v85: 97 pgs: 97 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 673 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:35:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:01 smithi110 ceph-mon[49387]: pgmap v85: 97 pgs: 97 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.8 MiB/s wr, 673 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:35:02.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 182608 9.37 MB/sec execute 356 sec latency 42.248 ms 2024-09-13T05:35:03.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 182904 9.37 MB/sec execute 357 sec latency 62.008 ms 2024-09-13T05:35:03.860 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:03 smithi053 ceph-mon[64292]: pgmap v86: 97 pgs: 97 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 487 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:35:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:03 smithi110 ceph-mon[49387]: pgmap v86: 97 pgs: 97 active+clean; 355 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.4 MiB/s wr, 487 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:35:04.382 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:04 smithi053 systemd[1]: Stopping Ceph osd.1 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:35:04.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 183270 9.37 MB/sec execute 358 sec latency 49.126 ms 2024-09-13T05:35:04.633 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:04 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[37057]: 2024-09-13T05:35:04.600+0000 7fb9f26b3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:35:04.633 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:04 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[37057]: 2024-09-13T05:35:04.600+0000 7fb9f26b3700 -1 osd.1 85 *** Got signal Terminated *** 2024-09-13T05:35:04.633 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:04 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[37057]: 2024-09-13T05:35:04.600+0000 7fb9f26b3700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:35:04.889 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:04 smithi053 ceph-mon[64292]: osd.1 marked itself down and dead 2024-09-13T05:35:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:04 smithi110 ceph-mon[49387]: osd.1 marked itself down and dead 2024-09-13T05:35:05.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 183423 9.35 MB/sec execute 359 sec latency 787.810 ms 2024-09-13T05:35:05.652 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:05 smithi053 podman[71139]: 2024-09-13 05:35:05.354598445 +0000 UTC m=+0.907684537 container died d22368ce725158f45799477217d8f95d74ce9d6a3c4be433d91d2f711382ce00 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, release=754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, ceph=True, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, RELEASE=HEAD, architecture=x86_64) 2024-09-13T05:35:05.908 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:05 smithi053 ceph-mon[64292]: pgmap v87: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 669 op/s 2024-09-13T05:35:05.908 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:05 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:35:05.908 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:05 smithi053 ceph-mon[64292]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T05:35:05.908 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:05 smithi053 podman[71139]: 2024-09-13 05:35:05.775572629 +0000 UTC m=+1.328658715 container cleanup d22368ce725158f45799477217d8f95d74ce9d6a3c4be433d91d2f711382ce00 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_BRANCH=HEAD, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.buildah.version=1.19.8, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.component=centos-stream-container) 2024-09-13T05:35:05.908 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:05 smithi053 bash[71139]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1 2024-09-13T05:35:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:05 smithi110 ceph-mon[49387]: pgmap v87: 97 pgs: 97 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 669 op/s 2024-09-13T05:35:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:05 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:35:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:05 smithi110 ceph-mon[49387]: osdmap e86: 8 total, 7 up, 8 in 2024-09-13T05:35:06.420 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:06 smithi053 podman[71155]: 2024-09-13 05:35:06.386673608 +0000 UTC m=+1.020867181 container remove d22368ce725158f45799477217d8f95d74ce9d6a3c4be433d91d2f711382ce00 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, ceph=True, release=754, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, RELEASE=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T05:35:06.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 183680 9.35 MB/sec execute 360 sec latency 1034.863 ms 2024-09-13T05:35:06.928 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:06 smithi053 ceph-mon[64292]: osdmap e87: 8 total, 7 up, 8 in 2024-09-13T05:35:06.928 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:06 smithi053 podman[71238]: 2024-09-13 05:35:06.81113994 +0000 UTC m=+0.023391659 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:35:06.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:06 smithi110 ceph-mon[49387]: osdmap e87: 8 total, 7 up, 8 in 2024-09-13T05:35:07.179 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:06 smithi053 podman[71238]: 2024-09-13 05:35:06.99967963 +0000 UTC m=+0.211931328 container create cbacb57749bf7c2bbb2d24f7c2cce58de99ad147bc5b70baa87f92832879d448 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T05:35:07.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 183932 9.34 MB/sec execute 361 sec latency 81.025 ms 2024-09-13T05:35:07.688 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:07 smithi053 podman[71238]: 2024-09-13 05:35:07.543060935 +0000 UTC m=+0.755312638 container init cbacb57749bf7c2bbb2d24f7c2cce58de99ad147bc5b70baa87f92832879d448 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T05:35:07.688 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:07 smithi053 podman[71238]: 2024-09-13 05:35:07.548454748 +0000 UTC m=+0.760706450 container start cbacb57749bf7c2bbb2d24f7c2cce58de99ad147bc5b70baa87f92832879d448 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:35:07.688 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:07 smithi053 podman[71238]: 2024-09-13 05:35:07.675969855 +0000 UTC m=+0.888221562 container attach cbacb57749bf7c2bbb2d24f7c2cce58de99ad147bc5b70baa87f92832879d448 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:35:07.954 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:07 smithi053 ceph-mon[64292]: pgmap v90: 97 pgs: 2 active+undersized, 12 stale+active+clean, 2 active+undersized+degraded, 81 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 596 op/s; 21/1575 objects degraded (1.333%) 2024-09-13T05:35:07.954 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:07 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 21/1575 objects degraded (1.333%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:07.954 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:07 smithi053 podman[71238]: 2024-09-13 05:35:07.881683476 +0000 UTC m=+1.093935175 container died cbacb57749bf7c2bbb2d24f7c2cce58de99ad147bc5b70baa87f92832879d448 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T05:35:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:07 smithi110 ceph-mon[49387]: pgmap v90: 97 pgs: 2 active+undersized, 12 stale+active+clean, 2 active+undersized+degraded, 81 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 596 op/s; 21/1575 objects degraded (1.333%) 2024-09-13T05:35:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:07 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 21/1575 objects degraded (1.333%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:08.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 184240 9.34 MB/sec execute 362 sec latency 70.874 ms 2024-09-13T05:35:08.987 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:08 smithi053 ceph-mon[64292]: pgmap v91: 97 pgs: 2 active+undersized, 12 stale+active+clean, 2 active+undersized+degraded, 81 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.2 MiB/s wr, 324 op/s; 21/1575 objects degraded (1.333%) 2024-09-13T05:35:08.987 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:08 smithi053 podman[71261]: 2024-09-13 05:35:08.851873859 +0000 UTC m=+0.953946894 container remove cbacb57749bf7c2bbb2d24f7c2cce58de99ad147bc5b70baa87f92832879d448 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T05:35:08.987 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:08 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T05:35:08.987 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:08 smithi053 systemd[1]: Stopped Ceph osd.1 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:35:08.987 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:08 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.1.service: Consumed 3min 14.669s CPU time. 2024-09-13T05:35:08.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:08 smithi110 ceph-mon[49387]: pgmap v91: 97 pgs: 2 active+undersized, 12 stale+active+clean, 2 active+undersized+degraded, 81 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.2 MiB/s wr, 324 op/s; 21/1575 objects degraded (1.333%) 2024-09-13T05:35:09.267 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:09 smithi053 systemd[1]: Starting Ceph osd.1 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:35:09.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 184478 9.34 MB/sec execute 363 sec latency 96.641 ms 2024-09-13T05:35:09.529 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:09 smithi053 podman[71356]: 2024-09-13 05:35:09.402300422 +0000 UTC m=+0.019859874 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:35:09.783 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:09 smithi053 podman[71356]: 2024-09-13 05:35:09.535839935 +0000 UTC m=+0.153399384 container create 0b6168ee073ba94013f8f1501cdc1aa8fbbbe1f6f4dd15903b66b347fb9e5d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731) 2024-09-13T05:35:10.034 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:09 smithi053 podman[71356]: 2024-09-13 05:35:09.898112435 +0000 UTC m=+0.515671887 container init 0b6168ee073ba94013f8f1501cdc1aa8fbbbe1f6f4dd15903b66b347fb9e5d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T05:35:10.034 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:09 smithi053 podman[71356]: 2024-09-13 05:35:09.903943404 +0000 UTC m=+0.521502857 container start 0b6168ee073ba94013f8f1501cdc1aa8fbbbe1f6f4dd15903b66b347fb9e5d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T05:35:10.034 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:10 smithi053 podman[71356]: 2024-09-13 05:35:10.006493316 +0000 UTC m=+0.624052780 container attach 0b6168ee073ba94013f8f1501cdc1aa8fbbbe1f6f4dd15903b66b347fb9e5d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731) 2024-09-13T05:35:10.285 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:10 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:10.285 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:10 smithi053 bash[71356]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:10.285 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:10 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:10.285 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:10 smithi053 bash[71356]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:10.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 184766 9.34 MB/sec execute 364 sec latency 47.414 ms 2024-09-13T05:35:11.289 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:11 smithi053 ceph-mon[64292]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 523 op/s; 180/1605 objects degraded (11.215%) 2024-09-13T05:35:11.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 185142 9.35 MB/sec execute 365 sec latency 55.025 ms 2024-09-13T05:35:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:11 smithi110 ceph-mon[49387]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.6 MiB/s wr, 523 op/s; 180/1605 objects degraded (11.215%) 2024-09-13T05:35:12.302 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:12 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 180/1623 objects degraded (11.091%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:12.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 185588 9.34 MB/sec execute 366 sec latency 52.229 ms 2024-09-13T05:35:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:12 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 180/1623 objects degraded (11.091%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:13.093 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:12 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T05:35:13.093 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:12 smithi053 bash[71356]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T05:35:13.093 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:12 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T05:35:13.093 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:12 smithi053 bash[71356]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T05:35:13.351 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T05:35:13.351 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 bash[71356]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 bash[71356]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 bash[71356]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 bash[71356]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate[71370]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 bash[71356]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T05:35:13.352 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:13 smithi053 podman[71356]: 2024-09-13 05:35:13.313312307 +0000 UTC m=+3.930871765 container died 0b6168ee073ba94013f8f1501cdc1aa8fbbbe1f6f4dd15903b66b347fb9e5d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T05:35:13.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 185856 9.33 MB/sec execute 367 sec latency 50.915 ms 2024-09-13T05:35:13.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-mon[64292]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.7 MiB/s wr, 253 op/s; 180/1623 objects degraded (11.091%) 2024-09-13T05:35:13.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:13.603 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:13 smithi110 ceph-mon[49387]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 349 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 2.7 MiB/s wr, 253 op/s; 180/1623 objects degraded (11.091%) 2024-09-13T05:35:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:14.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 186197 9.33 MB/sec execute 368 sec latency 63.429 ms 2024-09-13T05:35:14.623 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:14 smithi053 podman[71543]: 2024-09-13 05:35:14.539292286 +0000 UTC m=+1.217803112 container remove 0b6168ee073ba94013f8f1501cdc1aa8fbbbe1f6f4dd15903b66b347fb9e5d6a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:35:14.874 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:14 smithi053 podman[71612]: 2024-09-13 05:35:14.673250975 +0000 UTC m=+0.020100052 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:35:15.145 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:14 smithi053 podman[71612]: 2024-09-13 05:35:14.995518041 +0000 UTC m=+0.342367116 container create 025330b4adae5be123710d51080a6fecdc560f5a35bf336f71044fa2233b89d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T05:35:15.406 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:15 smithi053 podman[71612]: 2024-09-13 05:35:15.319984534 +0000 UTC m=+0.666833616 container init 025330b4adae5be123710d51080a6fecdc560f5a35bf336f71044fa2233b89d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:35:15.406 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:15 smithi053 podman[71612]: 2024-09-13 05:35:15.333020382 +0000 UTC m=+0.679869461 container start 025330b4adae5be123710d51080a6fecdc560f5a35bf336f71044fa2233b89d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:35:15.406 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:15 smithi053 ceph-osd[71631]: -- 172.21.15.53:0/1046543378 <== mon.0 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55deef72a1e0 con 0x55deef6be000 2024-09-13T05:35:15.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 186508 9.32 MB/sec execute 369 sec latency 61.965 ms 2024-09-13T05:35:15.657 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:15 smithi053 bash[71612]: 025330b4adae5be123710d51080a6fecdc560f5a35bf336f71044fa2233b89d0 2024-09-13T05:35:15.657 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:15 smithi053 systemd[1]: Started Ceph osd.1 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:35:15.657 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:15 smithi053 ceph-mon[64292]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 194/1683 objects degraded (11.527%) 2024-09-13T05:35:15.657 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:15.658 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:15.658 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:35:15.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:15 smithi110 ceph-mon[49387]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.7 MiB/s wr, 467 op/s; 194/1683 objects degraded (11.527%) 2024-09-13T05:35:15.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:15.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:15.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:35:16.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 186826 9.33 MB/sec execute 370 sec latency 69.983 ms 2024-09-13T05:35:17.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 187115 9.32 MB/sec execute 371 sec latency 53.822 ms 2024-09-13T05:35:17.685 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:17 smithi053 ceph-mon[64292]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 192/1662 objects degraded (11.552%) 2024-09-13T05:35:17.685 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:17 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 192/1662 objects degraded (11.552%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:17.686 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:17 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[71627]: 2024-09-13T05:35:17.663+0000 7f014570d640 -1 Falling back to public interface 2024-09-13T05:35:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:17 smithi110 ceph-mon[49387]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 192/1662 objects degraded (11.552%) 2024-09-13T05:35:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:17 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 192/1662 objects degraded (11.552%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:18.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 187431 9.32 MB/sec execute 372 sec latency 62.313 ms 2024-09-13T05:35:19.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 187727 9.32 MB/sec execute 373 sec latency 51.994 ms 2024-09-13T05:35:19.711 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:19 smithi053 ceph-mon[64292]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 366 op/s; 192/1662 objects degraded (11.552%) 2024-09-13T05:35:19.760 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:35:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:19 smithi110 ceph-mon[49387]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 366 op/s; 192/1662 objects degraded (11.552%) 2024-09-13T05:35:20.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 187947 9.31 MB/sec execute 374 sec latency 44.466 ms 2024-09-13T05:35:20.900 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:35:20.900 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (4m) 44s ago 14m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:35:20.901 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (93s) 44s ago 14m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:35:20.901 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (88s) 83s ago 14m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:35:20.901 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (4m) 44s ago 14m 43.1M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:35:20.901 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (8m) 44s ago 8m 37.7M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:35:20.901 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (8m) 83s ago 8m 16.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:35:20.901 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (3m) 44s ago 16m 511M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:35:20.902 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (3m) 83s ago 14m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:35:20.902 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (2m) 44s ago 16m 47.3M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:35:20.902 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (2m) 83s ago 14m 36.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:35:20.902 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (8m) 44s ago 8m 275M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:35:20.902 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (8m) 83s ago 8m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:35:20.902 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (4m) 44s ago 14m 10.4M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:35:20.903 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (4m) 83s ago 13m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:35:20.903 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (54s) 44s ago 13m 11.4M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:35:20.903 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 starting - - - 1937M 2024-09-13T05:35:20.903 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (12m) 44s ago 12m 411M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:35:20.903 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (11m) 44s ago 11m 541M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:35:20.903 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (11m) 83s ago 11m 527M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:35:20.904 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (11m) 83s ago 11m 337M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:35:20.904 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (10m) 83s ago 10m 525M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:35:20.904 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (10m) 83s ago 10m 431M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:35:20.904 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (3m) 44s ago 13m 48.7M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:35:21.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 188183 9.31 MB/sec execute 375 sec latency 62.261 ms 2024-09-13T05:35:21.698 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 1 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:35:21.699 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:21.700 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:35:21.700 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-13T05:35:21.700 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 5 2024-09-13T05:35:21.700 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:35:21.700 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:35:21.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:21 smithi053 ceph-mon[64292]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:21.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:21 smithi053 ceph-mon[64292]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 536 op/s; 163/1584 objects degraded (10.290%) 2024-09-13T05:35:21.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:21 smithi053 ceph-mon[64292]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:21 smithi110 ceph-mon[49387]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:21 smithi110 ceph-mon[49387]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.3 MiB/s wr, 536 op/s; 163/1584 objects degraded (10.290%) 2024-09-13T05:35:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:21 smithi110 ceph-mon[49387]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:22.321 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:35:22.321 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:35:22.321 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:35:22.321 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:35:22.321 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:35:22.321 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:35:22.321 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:35:22.322 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:35:22.322 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:35:22.322 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "7/23 daemons upgraded", 2024-09-13T05:35:22.322 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:35:22.322 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:35:22.322 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:35:22.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 188365 9.30 MB/sec execute 376 sec latency 57.076 ms 2024-09-13T05:35:22.757 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:22 smithi053 ceph-mon[64292]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:22.757 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:22 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1877568386' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:22.757 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:22 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 163/1584 objects degraded (10.290%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:22 smithi110 ceph-mon[49387]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:22 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1877568386' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:22 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 163/1584 objects degraded (10.290%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:23.048 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 166/1605 objects degraded (10.343%), 24 pgs degraded 2024-09-13T05:35:23.048 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: osd.1 (root=default,host=smithi053) is down 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 166/1605 objects degraded (10.343%), 24 pgs degraded 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.9 is active+undersized+degraded, acting [2,3] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-13T05:35:23.049 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-13T05:35:23.050 INFO:teuthology.orchestra.run.smithi053.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-13T05:35:23.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 188654 9.29 MB/sec execute 377 sec latency 66.936 ms 2024-09-13T05:35:23.764 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:23 smithi053 ceph-mon[64292]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 166/1605 objects degraded (10.343%) 2024-09-13T05:35:23.764 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:23 smithi053 ceph-mon[64292]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:23.764 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:23 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/4239238179' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:35:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:23 smithi110 ceph-mon[49387]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 347 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.3 MiB/s wr, 405 op/s; 166/1605 objects degraded (10.343%) 2024-09-13T05:35:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:23 smithi110 ceph-mon[49387]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:23 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/4239238179' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:35:24.468 INFO:teuthology.orchestra.run.smithi053.stdout: 5 189075 9.30 MB/sec execute 378 sec latency 40.064 ms 2024-09-13T05:35:24.521 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:24 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[71627]: 2024-09-13T05:35:24.428+0000 7f014570d640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-13T05:35:25.288 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:25 smithi053 ceph-mon[64292]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 170/1641 objects degraded (10.360%) 2024-09-13T05:35:25.288 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:25 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:25.288 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:25 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:25.288 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:25 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:25.288 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:25 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:25.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 189349 9.28 MB/sec execute 379 sec latency 57.211 ms 2024-09-13T05:35:25.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:25 smithi110 ceph-mon[49387]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 538 op/s; 170/1641 objects degraded (10.360%) 2024-09-13T05:35:25.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:25 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:25.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:25 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:25.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:25 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:25.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:25 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:26.060 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:25 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[71627]: 2024-09-13T05:35:25.827+0000 7f014570d640 -1 osd.1 85 log_to_monitors true 2024-09-13T05:35:26.311 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:26 smithi053 ceph-mon[64292]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T05:35:26.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 189702 9.28 MB/sec execute 380 sec latency 58.200 ms 2024-09-13T05:35:26.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:26 smithi110 ceph-mon[49387]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T05:35:26.832 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:35:26 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[71627]: 2024-09-13T05:35:26.591+0000 7f01394af640 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:35:27.355 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:27 smithi053 ceph-mon[64292]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 355 op/s; 168/1635 objects degraded (10.275%) 2024-09-13T05:35:27.355 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:27 smithi053 ceph-mon[64292]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T05:35:27.355 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:27 smithi053 ceph-mon[64292]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T05:35:27.355 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:27 smithi053 ceph-mon[64292]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:35:27.355 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:27 smithi053 ceph-mon[64292]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' 2024-09-13T05:35:27.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 189988 9.28 MB/sec execute 381 sec latency 53.969 ms 2024-09-13T05:35:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:27 smithi110 ceph-mon[49387]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 355 op/s; 168/1635 objects degraded (10.275%) 2024-09-13T05:35:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:27 smithi110 ceph-mon[49387]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T05:35:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:27 smithi110 ceph-mon[49387]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T05:35:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:27 smithi110 ceph-mon[49387]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:35:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:27 smithi110 ceph-mon[49387]: from='osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030]' entity='osd.1' 2024-09-13T05:35:28.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:28 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 168/1635 objects degraded (10.275%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:28 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:35:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:28 smithi053 ceph-mon[64292]: osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030] boot 2024-09-13T05:35:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:28 smithi053 ceph-mon[64292]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T05:35:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:28 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T05:35:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:28 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:28.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:28 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:28.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 190052 9.25 MB/sec execute 382 sec latency 909.265 ms 2024-09-13T05:35:28.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:28 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 168/1635 objects degraded (10.275%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:28.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:28 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:35:28.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:28 smithi110 ceph-mon[49387]: osd.1 [v2:172.21.15.53:6810/1602654030,v1:172.21.15.53:6811/1602654030] boot 2024-09-13T05:35:28.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:28 smithi110 ceph-mon[49387]: osdmap e89: 8 total, 8 up, 8 in 2024-09-13T05:35:28.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:28 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T05:35:28.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:28 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:28.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:28 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:29.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 190062 9.23 MB/sec execute 383 sec latency 1909.295 ms 2024-09-13T05:35:29.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:29 smithi053 ceph-mon[64292]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.2 MiB/s wr, 254 op/s; 168/1635 objects degraded (10.275%) 2024-09-13T05:35:29.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:29 smithi053 ceph-mon[64292]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T05:35:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:29 smithi110 ceph-mon[49387]: pgmap v103: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.2 MiB/s wr, 254 op/s; 168/1635 objects degraded (10.275%) 2024-09-13T05:35:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:29 smithi110 ceph-mon[49387]: osdmap e90: 8 total, 8 up, 8 in 2024-09-13T05:35:30.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 190298 9.23 MB/sec execute 384 sec latency 2210.844 ms 2024-09-13T05:35:30.735 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:30 smithi053 ceph-mon[64292]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T05:35:30.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:30 smithi110 ceph-mon[49387]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T05:35:31.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 190592 9.22 MB/sec execute 385 sec latency 48.250 ms 2024-09-13T05:35:31.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: pgmap v106: 97 pgs: 23 peering, 6 active+undersized, 10 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.9 MiB/s wr, 518 op/s; 66/1587 objects degraded (4.159%) 2024-09-13T05:35:31.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T05:35:31.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:35:31.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:35:31.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:35:31.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:31.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:31.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:31.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:31 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T05:35:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: pgmap v106: 97 pgs: 23 peering, 6 active+undersized, 10 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.9 MiB/s wr, 518 op/s; 66/1587 objects degraded (4.159%) 2024-09-13T05:35:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:31.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:31 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T05:35:32.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 190757 9.22 MB/sec execute 386 sec latency 859.553 ms 2024-09-13T05:35:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:32 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T05:35:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:32 smithi053 ceph-mon[64292]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T05:35:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:32 smithi053 ceph-mon[64292]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T05:35:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:32 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 66/1587 objects degraded (4.159%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:32.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:32 smithi053 ceph-mon[64292]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T05:35:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:32 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T05:35:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:32 smithi110 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-13T05:35:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:32 smithi110 ceph-mon[49387]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T05:35:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:32 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 66/1587 objects degraded (4.159%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:32.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:32 smithi110 ceph-mon[49387]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T05:35:33.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 190908 9.21 MB/sec execute 387 sec latency 917.537 ms 2024-09-13T05:35:33.631 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:33 smithi053 ceph-mon[64292]: pgmap v109: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 peering, 2 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.5 MiB/s wr, 552 op/s; 49/1581 objects degraded (3.099%); 22 B/s, 55 keys/s, 5 objects/s recovering 2024-09-13T05:35:33.631 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:33 smithi053 ceph-mon[64292]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T05:35:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:33 smithi110 ceph-mon[49387]: pgmap v109: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 23 peering, 2 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.5 MiB/s wr, 552 op/s; 49/1581 objects degraded (3.099%); 22 B/s, 55 keys/s, 5 objects/s recovering 2024-09-13T05:35:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:33 smithi110 ceph-mon[49387]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T05:35:34.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 191109 9.19 MB/sec execute 388 sec latency 695.758 ms 2024-09-13T05:35:35.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 191328 9.19 MB/sec execute 389 sec latency 572.711 ms 2024-09-13T05:35:35.682 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:35 smithi053 ceph-mon[64292]: pgmap v112: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+undersized+degraded, 89 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 MiB/s wr, 308 op/s; 38/1557 objects degraded (2.441%); 2.0 MiB/s, 55 keys/s, 46 objects/s recovering 2024-09-13T05:35:35.682 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:35 smithi053 ceph-mon[64292]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T05:35:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:35 smithi110 ceph-mon[49387]: pgmap v112: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 peering, 1 active+undersized+degraded, 89 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 MiB/s wr, 308 op/s; 38/1557 objects degraded (2.441%); 2.0 MiB/s, 55 keys/s, 46 objects/s recovering 2024-09-13T05:35:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:35 smithi110 ceph-mon[49387]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T05:35:36.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 191564 9.18 MB/sec execute 390 sec latency 78.375 ms 2024-09-13T05:35:37.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 191724 9.16 MB/sec execute 391 sec latency 84.022 ms 2024-09-13T05:35:37.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:37 smithi053 ceph-mon[64292]: pgmap v114: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 peering, 1 active+undersized+degraded, 90 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.0 MiB/s wr, 274 op/s; 10/1575 objects degraded (0.635%); 1.9 MiB/s, 43 objects/s recovering 2024-09-13T05:35:37.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:37 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 10/1575 objects degraded (0.635%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:37 smithi110 ceph-mon[49387]: pgmap v114: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 peering, 1 active+undersized+degraded, 90 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.0 MiB/s wr, 274 op/s; 10/1575 objects degraded (0.635%); 1.9 MiB/s, 43 objects/s recovering 2024-09-13T05:35:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:37 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 10/1575 objects degraded (0.635%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:35:38.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 192043 9.17 MB/sec execute 392 sec latency 53.707 ms 2024-09-13T05:35:39.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 192344 9.16 MB/sec execute 393 sec latency 45.373 ms 2024-09-13T05:35:39.769 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:39 smithi053 ceph-mon[64292]: pgmap v115: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 peering, 92 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.4 MiB/s wr, 212 op/s; 3/1575 objects degraded (0.190%); 1.4 MiB/s, 36 objects/s recovering 2024-09-13T05:35:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:39 smithi110 ceph-mon[49387]: pgmap v115: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 4 peering, 92 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.4 MiB/s wr, 212 op/s; 3/1575 objects degraded (0.190%); 1.4 MiB/s, 36 objects/s recovering 2024-09-13T05:35:40.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 192647 9.16 MB/sec execute 394 sec latency 79.108 ms 2024-09-13T05:35:40.795 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:40 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/1575 objects degraded (0.190%), 1 pg degraded) 2024-09-13T05:35:40.795 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:40 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:35:40.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:40 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/1575 objects degraded (0.190%), 1 pg degraded) 2024-09-13T05:35:40.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:40 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:35:41.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 192926 9.15 MB/sec execute 395 sec latency 82.416 ms 2024-09-13T05:35:41.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:41 smithi053 ceph-mon[64292]: pgmap v116: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 327 op/s; 1.1 MiB/s, 30 objects/s recovering 2024-09-13T05:35:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:41 smithi110 ceph-mon[49387]: pgmap v116: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 327 op/s; 1.1 MiB/s, 30 objects/s recovering 2024-09-13T05:35:42.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 193234 9.15 MB/sec execute 396 sec latency 44.006 ms 2024-09-13T05:35:43.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 193565 9.15 MB/sec execute 397 sec latency 43.609 ms 2024-09-13T05:35:43.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:43 smithi053 ceph-mon[64292]: pgmap v117: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.9 MiB/s wr, 307 op/s; 47 KiB/s, 7 objects/s recovering 2024-09-13T05:35:43.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:43.612 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:43 smithi110 ceph-mon[49387]: pgmap v117: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.9 MiB/s wr, 307 op/s; 47 KiB/s, 7 objects/s recovering 2024-09-13T05:35:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:44.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 193914 9.15 MB/sec execute 398 sec latency 43.022 ms 2024-09-13T05:35:45.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 194339 9.16 MB/sec execute 399 sec latency 31.350 ms 2024-09-13T05:35:45.638 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:45 smithi053 ceph-mon[64292]: pgmap v118: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 319 op/s; 40 KiB/s, 6 objects/s recovering 2024-09-13T05:35:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:45 smithi110 ceph-mon[49387]: pgmap v118: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.6 MiB/s wr, 319 op/s; 40 KiB/s, 6 objects/s recovering 2024-09-13T05:35:46.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 194769 9.17 MB/sec execute 400 sec latency 41.474 ms 2024-09-13T05:35:46.659 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T05:35:46.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T05:35:47.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 195006 9.16 MB/sec execute 401 sec latency 47.873 ms 2024-09-13T05:35:47.703 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:47 smithi053 ceph-mon[64292]: pgmap v119: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 390 op/s; 35 KiB/s, 5 objects/s recovering 2024-09-13T05:35:47.703 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:47 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T05:35:47.703 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:47 smithi053 ceph-mon[64292]: Upgrade: osd.2 is safe to restart 2024-09-13T05:35:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:47 smithi110 ceph-mon[49387]: pgmap v119: 97 pgs: 97 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.8 MiB/s wr, 390 op/s; 35 KiB/s, 5 objects/s recovering 2024-09-13T05:35:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:47 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T05:35:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:47 smithi110 ceph-mon[49387]: Upgrade: osd.2 is safe to restart 2024-09-13T05:35:48.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 195175 9.15 MB/sec execute 402 sec latency 69.634 ms 2024-09-13T05:35:48.768 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:48.768 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:48.768 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T05:35:48.768 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:48 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:35:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:48 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:48 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:48 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T05:35:48.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:48 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:35:49.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 195344 9.14 MB/sec execute 403 sec latency 63.898 ms 2024-09-13T05:35:49.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:49 smithi053 ceph-mon[64292]: pgmap v120: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 372 op/s; 0 B/s, 2 objects/s recovering 2024-09-13T05:35:49.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:49 smithi053 ceph-mon[64292]: Upgrade: Updating osd.2 2024-09-13T05:35:49.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:49 smithi053 ceph-mon[64292]: Deploying daemon osd.2 on smithi053 2024-09-13T05:35:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:49 smithi110 ceph-mon[49387]: pgmap v120: 97 pgs: 97 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 372 op/s; 0 B/s, 2 objects/s recovering 2024-09-13T05:35:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:49 smithi110 ceph-mon[49387]: Upgrade: Updating osd.2 2024-09-13T05:35:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:49 smithi110 ceph-mon[49387]: Deploying daemon osd.2 on smithi053 2024-09-13T05:35:50.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 195569 9.13 MB/sec execute 404 sec latency 72.706 ms 2024-09-13T05:35:51.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 195879 9.12 MB/sec execute 405 sec latency 44.815 ms 2024-09-13T05:35:51.862 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:51 smithi053 ceph-mon[64292]: pgmap v121: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 507 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:35:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:51 smithi110 ceph-mon[49387]: pgmap v121: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.0 MiB/s wr, 507 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T05:35:52.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 196117 9.12 MB/sec execute 406 sec latency 60.315 ms 2024-09-13T05:35:53.158 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:53 smithi053 systemd[1]: Stopping Ceph osd.2 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:35:53.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 196325 9.11 MB/sec execute 407 sec latency 64.353 ms 2024-09-13T05:35:53.552 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:35:53.664 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:53 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[40874]: 2024-09-13T05:35:53.511+0000 7fde58d41700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:35:53.664 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:53 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[40874]: 2024-09-13T05:35:53.511+0000 7fde58d41700 -1 osd.2 96 *** Got signal Terminated *** 2024-09-13T05:35:53.664 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:53 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[40874]: 2024-09-13T05:35:53.511+0000 7fde58d41700 -1 osd.2 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:35:53.922 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:53 smithi053 ceph-mon[64292]: pgmap v122: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-09-13T05:35:53.923 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:53 smithi053 ceph-mon[64292]: osd.2 marked itself down and dead 2024-09-13T05:35:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:53 smithi110 ceph-mon[49387]: pgmap v122: 97 pgs: 97 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.3 MiB/s wr, 421 op/s 2024-09-13T05:35:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:53 smithi110 ceph-mon[49387]: osd.2 marked itself down and dead 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (5m) 30s ago 15m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (2m) 30s ago 15m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (2m) 117s ago 14m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (5m) 30s ago 15m 43.6M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (9m) 30s ago 9m 38.0M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (9m) 117s ago 9m 16.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:35:54.355 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (4m) 30s ago 16m 516M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (3m) 117s ago 14m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (3m) 30s ago 16m 49.2M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (2m) 117s ago 14m 36.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (9m) 30s ago 9m 283M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (8m) 117s ago 8m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (5m) 30s ago 15m 10.3M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (4m) 117s ago 14m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (88s) 30s ago 13m 234M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (39s) 30s ago 13m 11.2M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (12m) 30s ago 12m 474M 1937M 17.2.0 e1d6a67b021e 81c1b142d206 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (12m) 30s ago 12m 597M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (12m) 117s ago 12m 527M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (11m) 117s ago 11m 337M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:35:54.356 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (11m) 117s ago 11m 525M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:35:54.357 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (10m) 117s ago 10m 431M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:35:54.357 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (3m) 30s ago 14m 53.0M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:35:54.460 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:54 smithi053 podman[73507]: 2024-09-13 05:35:54.427777479 +0000 UTC m=+1.186292919 container died 81c1b142d2067775e0a8c348729e481ac6f707f22287bd557dee4da9489d3ac7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, ceph=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vendor=Red Hat, Inc., architecture=x86_64, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T05:35:54.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 196481 9.09 MB/sec execute 408 sec latency 766.366 ms 2024-09-13T05:35:54.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:54 smithi053 ceph-mon[64292]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:54.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:54 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:35:54.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:54 smithi053 ceph-mon[64292]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T05:35:54.854 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:35:54.855 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T05:35:54.856 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 6 2024-09-13T05:35:54.856 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:35:54.856 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:35:54.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:54 smithi110 ceph-mon[49387]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:54.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:54 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:35:54.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:54 smithi110 ceph-mon[49387]: osdmap e97: 8 total, 7 up, 8 in 2024-09-13T05:35:55.231 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:55 smithi053 podman[73507]: 2024-09-13 05:35:55.050585752 +0000 UTC m=+1.809101180 container cleanup 81c1b142d2067775e0a8c348729e481ac6f707f22287bd557dee4da9489d3ac7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, vcs-type=git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, version=8, distribution-scope=public, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T05:35:55.231 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:55 smithi053 bash[73507]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2 2024-09-13T05:35:55.309 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:35:55.309 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:35:55.309 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:35:55.309 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:35:55.309 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "8/23 daemons upgraded", 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:35:55.310 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:35:55.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 196737 9.09 MB/sec execute 409 sec latency 1041.026 ms 2024-09-13T05:35:55.497 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:55 smithi053 podman[73591]: 2024-09-13 05:35:55.369973009 +0000 UTC m=+0.940524924 container remove 81c1b142d2067775e0a8c348729e481ac6f707f22287bd557dee4da9489d3ac7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.buildah.version=1.19.8, version=8, GIT_BRANCH=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public) 2024-09-13T05:35:55.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:55 smithi053 ceph-mon[64292]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:55.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:55 smithi053 ceph-mon[64292]: pgmap v124: 97 pgs: 6 stale+active+clean, 91 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 478 op/s 2024-09-13T05:35:55.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:55 smithi053 ceph-mon[64292]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:55.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:55 smithi053 ceph-mon[64292]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T05:35:55.766 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:55 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1179667739' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:55.905 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_WARN 1 osds down 2024-09-13T05:35:55.905 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T05:35:55.905 INFO:teuthology.orchestra.run.smithi053.stdout: osd.2 (root=default,host=smithi053) is down 2024-09-13T05:35:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:55 smithi110 ceph-mon[49387]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:55 smithi110 ceph-mon[49387]: pgmap v124: 97 pgs: 6 stale+active+clean, 91 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 478 op/s 2024-09-13T05:35:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:55 smithi110 ceph-mon[49387]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:55 smithi110 ceph-mon[49387]: osdmap e98: 8 total, 7 up, 8 in 2024-09-13T05:35:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:55 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1179667739' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:35:56.020 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:55 smithi053 podman[73739]: 2024-09-13 05:35:55.880344814 +0000 UTC m=+0.021671818 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:35:56.021 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:56 smithi053 podman[73739]: 2024-09-13 05:35:56.004744947 +0000 UTC m=+0.146071934 container create 1e5265872bcad1fcf5b17d67cfcc4693050abb0280d9355618f0151645b2e582 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:35:56.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 196972 9.08 MB/sec execute 410 sec latency 57.906 ms 2024-09-13T05:35:56.800 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:56 smithi053 ceph-mon[64292]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:56.800 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:56 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3493546345' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:35:56.800 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:56 smithi053 podman[73739]: 2024-09-13 05:35:56.570146651 +0000 UTC m=+0.711473644 container init 1e5265872bcad1fcf5b17d67cfcc4693050abb0280d9355618f0151645b2e582 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-13T05:35:56.800 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:56 smithi053 podman[73739]: 2024-09-13 05:35:56.577680835 +0000 UTC m=+0.719007829 container start 1e5265872bcad1fcf5b17d67cfcc4693050abb0280d9355618f0151645b2e582 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:35:56.800 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:56 smithi053 podman[73739]: 2024-09-13 05:35:56.751386276 +0000 UTC m=+0.892713276 container attach 1e5265872bcad1fcf5b17d67cfcc4693050abb0280d9355618f0151645b2e582 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:35:56.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:56 smithi110 ceph-mon[49387]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:35:56.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:56 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3493546345' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:35:57.066 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:56 smithi053 podman[73739]: 2024-09-13 05:35:56.903981488 +0000 UTC m=+1.045308485 container died 1e5265872bcad1fcf5b17d67cfcc4693050abb0280d9355618f0151645b2e582 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T05:35:57.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 197258 9.07 MB/sec execute 411 sec latency 73.520 ms 2024-09-13T05:35:57.823 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:57 smithi053 ceph-mon[64292]: pgmap v126: 97 pgs: 1 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 89 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 441 op/s; 13/1578 objects degraded (0.824%) 2024-09-13T05:35:57.823 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:57 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 13/1578 objects degraded (0.824%), 1 pg degraded (PG_DEGRADED) 2024-09-13T05:35:57.823 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:57.823 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:57.823 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:57 smithi053 podman[73764]: 2024-09-13 05:35:57.818410502 +0000 UTC m=+0.898066190 container remove 1e5265872bcad1fcf5b17d67cfcc4693050abb0280d9355618f0151645b2e582 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS) 2024-09-13T05:35:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:57 smithi110 ceph-mon[49387]: pgmap v126: 97 pgs: 1 active+undersized, 6 stale+active+clean, 1 active+undersized+degraded, 89 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.8 MiB/s wr, 441 op/s; 13/1578 objects degraded (0.824%) 2024-09-13T05:35:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:57 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 13/1578 objects degraded (0.824%), 1 pg degraded (PG_DEGRADED) 2024-09-13T05:35:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:35:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:35:58.091 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:57 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T05:35:58.091 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:57 smithi053 systemd[1]: Stopped Ceph osd.2 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:35:58.091 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:57 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 32.987s CPU time. 2024-09-13T05:35:58.352 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:58 smithi053 systemd[1]: Starting Ceph osd.2 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:35:58.469 INFO:teuthology.orchestra.run.smithi053.stdout: 5 197555 9.07 MB/sec execute 412 sec latency 58.592 ms 2024-09-13T05:35:58.613 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:58 smithi053 podman[73856]: 2024-09-13 05:35:58.341372817 +0000 UTC m=+0.021586075 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:35:58.613 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:58 smithi053 podman[73856]: 2024-09-13 05:35:58.485759773 +0000 UTC m=+0.165973004 container create 4910805ba0558ebabc581a4f1d721eaef19fb1cd3925f002914ba658b21ca483 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T05:35:59.121 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:58 smithi053 podman[73856]: 2024-09-13 05:35:58.932092364 +0000 UTC m=+0.612305614 container init 4910805ba0558ebabc581a4f1d721eaef19fb1cd3925f002914ba658b21ca483 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T05:35:59.121 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:58 smithi053 podman[73856]: 2024-09-13 05:35:58.941710578 +0000 UTC m=+0.621923819 container start 4910805ba0558ebabc581a4f1d721eaef19fb1cd3925f002914ba658b21ca483 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2) 2024-09-13T05:35:59.121 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:59 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:59.121 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:59 smithi053 bash[73856]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:59.121 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:59 smithi053 podman[73856]: 2024-09-13 05:35:59.107740375 +0000 UTC m=+0.787953610 container attach 4910805ba0558ebabc581a4f1d721eaef19fb1cd3925f002914ba658b21ca483 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:35:59.122 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:59 smithi053 bash[73856]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:59.122 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:35:59 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:35:59.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 197821 9.07 MB/sec execute 413 sec latency 75.176 ms 2024-09-13T05:35:59.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:35:59 smithi053 ceph-mon[64292]: pgmap v127: 97 pgs: 4 active+undersized, 4 stale+active+clean, 5 active+undersized+degraded, 84 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.4 MiB/s wr, 236 op/s; 47/1578 objects degraded (2.978%) 2024-09-13T05:35:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:35:59 smithi110 ceph-mon[49387]: pgmap v127: 97 pgs: 4 active+undersized, 4 stale+active+clean, 5 active+undersized+degraded, 84 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.4 MiB/s wr, 236 op/s; 47/1578 objects degraded (2.978%) 2024-09-13T05:36:00.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 198067 9.07 MB/sec execute 414 sec latency 62.254 ms 2024-09-13T05:36:00.947 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:00 smithi053 ceph-mon[64292]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.6 MiB/s wr, 408 op/s; 152/1527 objects degraded (9.954%) 2024-09-13T05:36:01.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:00 smithi110 ceph-mon[49387]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 3.6 MiB/s wr, 408 op/s; 152/1527 objects degraded (9.954%) 2024-09-13T05:36:01.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 198380 9.07 MB/sec execute 415 sec latency 47.298 ms 2024-09-13T05:36:02.225 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 152/1542 objects degraded (9.857%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:02.225 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T05:36:02.225 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 bash[73856]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T05:36:02.225 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T05:36:02.225 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 bash[73856]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T05:36:02.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 198795 9.08 MB/sec execute 416 sec latency 66.238 ms 2024-09-13T05:36:02.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:02 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 152/1542 objects degraded (9.857%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:02.773 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T05:36:02.773 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 bash[73856]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 bash[73856]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 bash[73856]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 bash[73856]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate[73870]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 bash[73856]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T05:36:02.774 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:02 smithi053 podman[73856]: 2024-09-13 05:36:02.551675146 +0000 UTC m=+4.231888381 container died 4910805ba0558ebabc581a4f1d721eaef19fb1cd3925f002914ba658b21ca483 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T05:36:03.295 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:03 smithi053 ceph-mon[64292]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.5 MiB/s wr, 237 op/s; 152/1542 objects degraded (9.857%) 2024-09-13T05:36:03.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 198969 9.07 MB/sec execute 417 sec latency 96.976 ms 2024-09-13T05:36:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:03 smithi110 ceph-mon[49387]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 350 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.8 KiB/s rd, 2.5 MiB/s wr, 237 op/s; 152/1542 objects degraded (9.857%) 2024-09-13T05:36:03.801 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:03 smithi053 podman[74018]: 2024-09-13 05:36:03.549287625 +0000 UTC m=+0.989028598 container remove 4910805ba0558ebabc581a4f1d721eaef19fb1cd3925f002914ba658b21ca483 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-13T05:36:03.801 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:03 smithi053 podman[74076]: 2024-09-13 05:36:03.683812787 +0000 UTC m=+0.021338841 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:36:04.052 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:03 smithi053 podman[74076]: 2024-09-13 05:36:03.80093346 +0000 UTC m=+0.138459503 container create 685d12d14368132f076f30e23a5295d0a95182d04a641def87155d0840146cd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2) 2024-09-13T05:36:04.249 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:04 smithi053 podman[74076]: 2024-09-13 05:36:04.164288016 +0000 UTC m=+0.501814060 container init 685d12d14368132f076f30e23a5295d0a95182d04a641def87155d0840146cd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux ) 2024-09-13T05:36:04.249 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:04 smithi053 podman[74076]: 2024-09-13 05:36:04.16922321 +0000 UTC m=+0.506749257 container start 685d12d14368132f076f30e23a5295d0a95182d04a641def87155d0840146cd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T05:36:04.249 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:04 smithi053 ceph-osd[74094]: -- 172.21.15.53:0/635117635 <== mon.0 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5622eb30a1e0 con 0x5622ea515c00 2024-09-13T05:36:04.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 199151 9.06 MB/sec execute 418 sec latency 79.373 ms 2024-09-13T05:36:04.567 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:04 smithi053 bash[74076]: 685d12d14368132f076f30e23a5295d0a95182d04a641def87155d0840146cd4 2024-09-13T05:36:04.567 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:04 smithi053 systemd[1]: Started Ceph osd.2 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:36:05.470 INFO:teuthology.orchestra.run.smithi053.stdout: 5 199329 9.05 MB/sec execute 419 sec latency 82.140 ms 2024-09-13T05:36:05.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:05 smithi053 ceph-mon[64292]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 380 op/s; 158/1632 objects degraded (9.681%) 2024-09-13T05:36:05.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:05.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:05.615 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:36:05.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:05 smithi110 ceph-mon[49387]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 349 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.0 MiB/s wr, 380 op/s; 158/1632 objects degraded (9.681%) 2024-09-13T05:36:05.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:05.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:05.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:36:06.376 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:06 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[74090]: 2024-09-13T05:36:06.206+0000 7f23fbd0d640 -1 Falling back to public interface 2024-09-13T05:36:06.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 199565 9.04 MB/sec execute 420 sec latency 75.326 ms 2024-09-13T05:36:07.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 199825 9.03 MB/sec execute 421 sec latency 64.112 ms 2024-09-13T05:36:07.656 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:07 smithi053 ceph-mon[64292]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.8 MiB/s wr, 349 op/s; 159/1623 objects degraded (9.797%) 2024-09-13T05:36:07.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:07 smithi110 ceph-mon[49387]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.8 MiB/s wr, 349 op/s; 159/1623 objects degraded (9.797%) 2024-09-13T05:36:08.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 200056 9.02 MB/sec execute 422 sec latency 61.628 ms 2024-09-13T05:36:08.639 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:08 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 159/1623 objects degraded (9.797%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:08.640 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:08.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:08 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 159/1623 objects degraded (9.797%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:08.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:09.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 200352 9.02 MB/sec execute 423 sec latency 53.618 ms 2024-09-13T05:36:09.694 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:09 smithi053 ceph-mon[64292]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.5 MiB/s wr, 324 op/s; 162/1638 objects degraded (9.890%) 2024-09-13T05:36:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:09 smithi110 ceph-mon[49387]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.5 MiB/s wr, 324 op/s; 162/1638 objects degraded (9.890%) 2024-09-13T05:36:10.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 200672 9.01 MB/sec execute 424 sec latency 41.781 ms 2024-09-13T05:36:11.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 201132 9.02 MB/sec execute 425 sec latency 86.417 ms 2024-09-13T05:36:11.739 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:11 smithi053 ceph-mon[64292]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 444 op/s; 169/1659 objects degraded (10.187%) 2024-09-13T05:36:11.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:11 smithi110 ceph-mon[49387]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 352 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 444 op/s; 169/1659 objects degraded (10.187%) 2024-09-13T05:36:12.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 201501 9.02 MB/sec execute 426 sec latency 43.403 ms 2024-09-13T05:36:12.684 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:12 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[74090]: 2024-09-13T05:36:12.602+0000 7f23fbd0d640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-13T05:36:12.939 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:12 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 169/1659 objects degraded (10.187%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:12.939 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:12.939 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:12.939 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:12.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:12 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 169/1659 objects degraded (10.187%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:12.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:12.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:12.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:13.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 201899 9.04 MB/sec execute 427 sec latency 40.013 ms 2024-09-13T05:36:13.704 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:13 smithi053 ceph-mon[64292]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 339 op/s; 169/1656 objects degraded (10.205%) 2024-09-13T05:36:13.704 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:13.704 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:13.704 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:13 smithi053 ceph-mon[64292]: from='osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T05:36:13.704 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[74090]: 2024-09-13T05:36:13.595+0000 7f23fbd0d640 -1 osd.2 96 log_to_monitors true 2024-09-13T05:36:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:13 smithi110 ceph-mon[49387]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 339 op/s; 169/1656 objects degraded (10.205%) 2024-09-13T05:36:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:13 smithi110 ceph-mon[49387]: from='osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T05:36:14.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 202122 9.03 MB/sec execute 428 sec latency 55.910 ms 2024-09-13T05:36:14.762 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:36:14 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[74090]: 2024-09-13T05:36:14.561+0000 7f23f3ab7640 -1 osd.2 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:36:15.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 202411 9.04 MB/sec execute 429 sec latency 59.325 ms 2024-09-13T05:36:15.515 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:15 smithi053 ceph-mon[64292]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 503 op/s; 164/1668 objects degraded (9.832%) 2024-09-13T05:36:15.516 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:15 smithi053 ceph-mon[64292]: from='osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T05:36:15.516 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:15 smithi053 ceph-mon[64292]: osdmap e99: 8 total, 7 up, 8 in 2024-09-13T05:36:15.516 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:15 smithi053 ceph-mon[64292]: from='osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:36:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:15 smithi110 ceph-mon[49387]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 503 op/s; 164/1668 objects degraded (9.832%) 2024-09-13T05:36:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:15 smithi110 ceph-mon[49387]: from='osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T05:36:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:15 smithi110 ceph-mon[49387]: osdmap e99: 8 total, 7 up, 8 in 2024-09-13T05:36:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:15 smithi110 ceph-mon[49387]: from='osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:36:16.473 INFO:teuthology.orchestra.run.smithi053.stdout: 5 202494 9.02 MB/sec execute 430 sec latency 921.257 ms 2024-09-13T05:36:16.643 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:16 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:36:16.643 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:16 smithi053 ceph-mon[64292]: osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761] boot 2024-09-13T05:36:16.643 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:16 smithi053 ceph-mon[64292]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T05:36:16.643 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:16 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T05:36:16.643 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:16 smithi053 ceph-mon[64292]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T05:36:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:16 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:36:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:16 smithi110 ceph-mon[49387]: osd.2 [v2:172.21.15.53:6818/2946259761,v1:172.21.15.53:6819/2946259761] boot 2024-09-13T05:36:16.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:16 smithi110 ceph-mon[49387]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T05:36:16.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:16 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T05:36:16.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:16 smithi110 ceph-mon[49387]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T05:36:17.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 202504 9.00 MB/sec execute 431 sec latency 1699.259 ms 2024-09-13T05:36:17.768 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:17 smithi053 ceph-mon[64292]: pgmap v138: 97 pgs: 2 peering, 6 active+undersized, 18 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 515 op/s; 147/1650 objects degraded (8.909%) 2024-09-13T05:36:17.769 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:17 smithi053 ceph-mon[64292]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T05:36:17.769 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:17 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 147/1650 objects degraded (8.909%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:17 smithi110 ceph-mon[49387]: pgmap v138: 97 pgs: 2 peering, 6 active+undersized, 18 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 515 op/s; 147/1650 objects degraded (8.909%) 2024-09-13T05:36:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:17 smithi110 ceph-mon[49387]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T05:36:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:17 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 147/1650 objects degraded (8.909%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:18.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 202699 8.99 MB/sec execute 432 sec latency 1840.039 ms 2024-09-13T05:36:19.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 202803 8.97 MB/sec execute 433 sec latency 751.323 ms 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: pgmap v141: 97 pgs: 7 peering, 4 active+undersized, 15 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s wr, 48 op/s; 121/1647 objects degraded (7.347%) 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.809 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:36:19.810 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:19.810 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:19.810 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:19.810 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T05:36:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: pgmap v141: 97 pgs: 7 peering, 4 active+undersized, 15 active+undersized+degraded, 71 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.3 MiB/s wr, 48 op/s; 121/1647 objects degraded (7.347%) 2024-09-13T05:36:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: osdmap e103: 8 total, 8 up, 8 in 2024-09-13T05:36:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T05:36:20.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 203061 8.96 MB/sec execute 434 sec latency 842.268 ms 2024-09-13T05:36:20.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:20 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T05:36:20.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:20 smithi053 ceph-mon[64292]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T05:36:20.602 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:20 smithi053 ceph-mon[64292]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T05:36:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:20 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T05:36:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:20 smithi110 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T05:36:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:20 smithi110 ceph-mon[49387]: osdmap e104: 8 total, 8 up, 8 in 2024-09-13T05:36:21.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 203214 8.95 MB/sec execute 435 sec latency 719.649 ms 2024-09-13T05:36:21.627 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:21 smithi053 ceph-mon[64292]: pgmap v144: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 7 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 331 op/s; 56/1548 objects degraded (3.618%); 1.0 MiB/s, 11 objects/s recovering 2024-09-13T05:36:21.627 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:21 smithi053 ceph-mon[64292]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T05:36:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:21 smithi110 ceph-mon[49387]: pgmap v144: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 7 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 331 op/s; 56/1548 objects degraded (3.618%); 1.0 MiB/s, 11 objects/s recovering 2024-09-13T05:36:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:21 smithi110 ceph-mon[49387]: osdmap e105: 8 total, 8 up, 8 in 2024-09-13T05:36:22.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 203412 8.94 MB/sec execute 436 sec latency 859.414 ms 2024-09-13T05:36:22.646 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:22 smithi053 ceph-mon[64292]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T05:36:22.646 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:22 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 56/1548 objects degraded (3.618%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:22 smithi110 ceph-mon[49387]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T05:36:22.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:22 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 56/1548 objects degraded (3.618%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:23.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 203627 8.93 MB/sec execute 437 sec latency 67.111 ms 2024-09-13T05:36:23.688 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:23 smithi053 ceph-mon[64292]: pgmap v147: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 336 op/s; 56/1551 objects degraded (3.611%); 1.1 MiB/s, 27 objects/s recovering 2024-09-13T05:36:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:23 smithi110 ceph-mon[49387]: pgmap v147: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 336 op/s; 56/1551 objects degraded (3.611%); 1.1 MiB/s, 27 objects/s recovering 2024-09-13T05:36:24.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 203857 8.92 MB/sec execute 438 sec latency 60.053 ms 2024-09-13T05:36:24.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:24 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/1551 objects degraded (3.611%), 3 pgs degraded) 2024-09-13T05:36:24.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:24 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:36:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:24 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/1551 objects degraded (3.611%), 3 pgs degraded) 2024-09-13T05:36:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:24 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:36:25.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 204186 8.92 MB/sec execute 439 sec latency 63.770 ms 2024-09-13T05:36:25.732 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:25 smithi053 ceph-mon[64292]: pgmap v148: 97 pgs: 1 peering, 96 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.1 MiB/s wr, 254 op/s; 101 KiB/s, 24 objects/s recovering 2024-09-13T05:36:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:25 smithi110 ceph-mon[49387]: pgmap v148: 97 pgs: 1 peering, 96 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 2.1 MiB/s wr, 254 op/s; 101 KiB/s, 24 objects/s recovering 2024-09-13T05:36:26.370 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:36:26.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 204396 8.91 MB/sec execute 440 sec latency 46.696 ms 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (5m) 14s ago 15m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (2m) 14s ago 15m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (2m) 2m ago 15m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (5m) 14s ago 15m 44.2M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (9m) 14s ago 9m 38.0M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (9m) 2m ago 9m 16.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (5m) 14s ago 17m 517M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:36:27.247 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (4m) 2m ago 15m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (3m) 14s ago 17m 50.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (3m) 2m ago 15m 36.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (9m) 14s ago 9m 284M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (9m) 2m ago 9m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (5m) 14s ago 15m 9.96M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (5m) 2m ago 15m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (2m) 14s ago 14m 282M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (72s) 14s ago 13m 220M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (23s) 14s ago 13m 12.1M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (12m) 14s ago 12m 389M 1937M 17.2.0 e1d6a67b021e 7bfc6396fd09 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (12m) 2m ago 12m 527M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (12m) 2m ago 12m 337M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:36:27.248 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (11m) 2m ago 11m 525M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:36:27.249 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (11m) 2m ago 11m 431M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:36:27.249 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (4m) 14s ago 14m 53.2M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:36:27.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 204693 8.91 MB/sec execute 441 sec latency 52.254 ms 2024-09-13T05:36:27.778 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 3 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:36:27.779 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:36:27.780 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T05:36:27.780 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 7 2024-09-13T05:36:27.780 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:36:27.780 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:36:27.790 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:27 smithi053 ceph-mon[64292]: pgmap v149: 97 pgs: 1 peering, 96 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.2 MiB/s wr, 237 op/s; 70 KiB/s, 20 objects/s recovering 2024-09-13T05:36:27.790 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:27 smithi053 ceph-mon[64292]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:27.790 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:27.790 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:27 smithi110 ceph-mon[49387]: pgmap v149: 97 pgs: 1 peering, 96 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.2 MiB/s wr, 237 op/s; 70 KiB/s, 20 objects/s recovering 2024-09-13T05:36:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:27 smithi110 ceph-mon[49387]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:36:28.232 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "9/23 daemons upgraded", 2024-09-13T05:36:28.233 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:36:28.233 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:36:28.233 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:36:28.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 204964 8.91 MB/sec execute 442 sec latency 80.467 ms 2024-09-13T05:36:28.759 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:36:28.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:28 smithi053 ceph-mon[64292]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:28.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:28 smithi053 ceph-mon[64292]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:28.821 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:28 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/2701502561' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:28.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:28 smithi110 ceph-mon[49387]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:28.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:28 smithi110 ceph-mon[49387]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:28.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:28 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/2701502561' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:36:29.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 205200 8.91 MB/sec execute 443 sec latency 44.481 ms 2024-09-13T05:36:29.860 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:29 smithi053 ceph-mon[64292]: pgmap v150: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.9 MiB/s wr, 202 op/s; 57 KiB/s, 16 objects/s recovering 2024-09-13T05:36:29.860 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:29 smithi053 ceph-mon[64292]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:29.860 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:29 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/4018479304' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:36:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:29 smithi110 ceph-mon[49387]: pgmap v150: 97 pgs: 97 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.9 MiB/s wr, 202 op/s; 57 KiB/s, 16 objects/s recovering 2024-09-13T05:36:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:29 smithi110 ceph-mon[49387]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:36:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:29 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/4018479304' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:36:30.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 205510 8.92 MB/sec execute 444 sec latency 58.130 ms 2024-09-13T05:36:31.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 205796 8.91 MB/sec execute 445 sec latency 49.100 ms 2024-09-13T05:36:31.943 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:31 smithi053 ceph-mon[64292]: pgmap v151: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.0 MiB/s wr, 341 op/s; 48 KiB/s, 14 objects/s recovering 2024-09-13T05:36:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:31 smithi110 ceph-mon[49387]: pgmap v151: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.0 MiB/s wr, 341 op/s; 48 KiB/s, 14 objects/s recovering 2024-09-13T05:36:32.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 206023 8.91 MB/sec execute 446 sec latency 48.797 ms 2024-09-13T05:36:33.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 206288 8.91 MB/sec execute 447 sec latency 70.307 ms 2024-09-13T05:36:33.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:33 smithi053 ceph-mon[64292]: pgmap v152: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.3 MiB/s wr, 358 op/s; 0 B/s, 6 objects/s recovering 2024-09-13T05:36:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:33 smithi110 ceph-mon[49387]: pgmap v152: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.3 MiB/s wr, 358 op/s; 0 B/s, 6 objects/s recovering 2024-09-13T05:36:34.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 206521 8.90 MB/sec execute 448 sec latency 53.994 ms 2024-09-13T05:36:34.874 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:34 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T05:36:35.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:34 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T05:36:35.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 206756 8.88 MB/sec execute 449 sec latency 60.513 ms 2024-09-13T05:36:35.761 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:35 smithi053 ceph-mon[64292]: pgmap v153: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 0 B/s, 5 objects/s recovering 2024-09-13T05:36:35.761 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:35 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T05:36:35.761 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:35 smithi053 ceph-mon[64292]: Upgrade: osd.3 is safe to restart 2024-09-13T05:36:36.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:35 smithi110 ceph-mon[49387]: pgmap v153: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 0 B/s, 5 objects/s recovering 2024-09-13T05:36:36.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:35 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T05:36:36.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:35 smithi110 ceph-mon[49387]: Upgrade: osd.3 is safe to restart 2024-09-13T05:36:36.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 206999 8.88 MB/sec execute 450 sec latency 87.820 ms 2024-09-13T05:36:36.779 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:36 smithi053 ceph-mon[64292]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.4 MiB/s wr, 282 op/s 2024-09-13T05:36:37.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:36 smithi110 ceph-mon[49387]: pgmap v154: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.4 MiB/s wr, 282 op/s 2024-09-13T05:36:37.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 207273 8.87 MB/sec execute 451 sec latency 50.823 ms 2024-09-13T05:36:38.051 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:37 smithi053 ceph-mon[64292]: Upgrade: Updating osd.3 2024-09-13T05:36:38.051 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:38.051 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T05:36:38.051 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:36:38.051 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:37 smithi053 ceph-mon[64292]: Deploying daemon osd.3 on smithi053 2024-09-13T05:36:38.051 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:37 smithi110 ceph-mon[49387]: Upgrade: Updating osd.3 2024-09-13T05:36:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T05:36:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:36:38.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:37 smithi110 ceph-mon[49387]: Deploying daemon osd.3 on smithi053 2024-09-13T05:36:38.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:38.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 207533 8.86 MB/sec execute 452 sec latency 67.081 ms 2024-09-13T05:36:39.092 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:38 smithi053 ceph-mon[64292]: pgmap v155: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.4 MiB/s wr, 317 op/s 2024-09-13T05:36:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:38 smithi110 ceph-mon[49387]: pgmap v155: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.4 MiB/s wr, 317 op/s 2024-09-13T05:36:39.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 207875 8.86 MB/sec execute 453 sec latency 71.890 ms 2024-09-13T05:36:40.474 INFO:teuthology.orchestra.run.smithi053.stdout: 5 208102 8.86 MB/sec execute 454 sec latency 59.117 ms 2024-09-13T05:36:41.215 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:41 smithi053 ceph-mon[64292]: pgmap v156: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 457 op/s 2024-09-13T05:36:41.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 208319 8.86 MB/sec execute 455 sec latency 51.540 ms 2024-09-13T05:36:41.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:41 smithi110 ceph-mon[49387]: pgmap v156: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.3 MiB/s wr, 457 op/s 2024-09-13T05:36:41.494 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:41 smithi053 systemd[1]: Stopping Ceph osd.3 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:36:41.747 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:41 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[44687]: 2024-09-13T05:36:41.650+0000 7fd3946c3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:36:41.747 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:41 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[44687]: 2024-09-13T05:36:41.650+0000 7fd3946c3700 -1 osd.3 106 *** Got signal Terminated *** 2024-09-13T05:36:41.747 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:41 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[44687]: 2024-09-13T05:36:41.650+0000 7fd3946c3700 -1 osd.3 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:36:42.263 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:42 smithi053 ceph-mon[64292]: osd.3 marked itself down and dead 2024-09-13T05:36:42.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 208501 8.85 MB/sec execute 456 sec latency 304.692 ms 2024-09-13T05:36:42.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:42 smithi110 ceph-mon[49387]: osd.3 marked itself down and dead 2024-09-13T05:36:43.109 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:42 smithi053 podman[76011]: 2024-09-13 05:36:42.819967473 +0000 UTC m=+1.366643156 container died 7bfc6396fd09cdae6458e98240f2a51837a08fbd510667af4426cec4b1e3350e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, ceph=True, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , name=centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, release=754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git) 2024-09-13T05:36:43.371 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:43 smithi053 ceph-mon[64292]: pgmap v157: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 341 op/s 2024-09-13T05:36:43.371 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:43 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:36:43.371 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:43 smithi053 ceph-mon[64292]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T05:36:43.371 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:43.371 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:43.372 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:43 smithi053 podman[76011]: 2024-09-13 05:36:43.232912571 +0000 UTC m=+1.779588254 container cleanup 7bfc6396fd09cdae6458e98240f2a51837a08fbd510667af4426cec4b1e3350e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., architecture=x86_64) 2024-09-13T05:36:43.372 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:43 smithi053 bash[76011]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3 2024-09-13T05:36:43.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 208605 8.83 MB/sec execute 457 sec latency 1005.298 ms 2024-09-13T05:36:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:43 smithi110 ceph-mon[49387]: pgmap v157: 97 pgs: 97 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.5 MiB/s wr, 341 op/s 2024-09-13T05:36:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:43 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:36:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:43 smithi110 ceph-mon[49387]: osdmap e107: 8 total, 7 up, 8 in 2024-09-13T05:36:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:43.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:43.623 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:43 smithi053 podman[76027]: 2024-09-13 05:36:43.561406229 +0000 UTC m=+0.732479957 container remove 7bfc6396fd09cdae6458e98240f2a51837a08fbd510667af4426cec4b1e3350e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T05:36:44.126 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:44 smithi053 podman[76116]: 2024-09-13 05:36:44.009984759 +0000 UTC m=+0.027926989 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:36:44.385 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:44 smithi053 ceph-mon[64292]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T05:36:44.386 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:44 smithi053 podman[76116]: 2024-09-13 05:36:44.1432926 +0000 UTC m=+0.161234824 container create 5821db7e190181ac053928cc5abc86f3b75feede597a15f6ea9f90a4002d8bee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:36:44.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 208990 8.85 MB/sec execute 458 sec latency 47.305 ms 2024-09-13T05:36:44.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:44 smithi110 ceph-mon[49387]: osdmap e108: 8 total, 7 up, 8 in 2024-09-13T05:36:44.650 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:44 smithi053 podman[76116]: 2024-09-13 05:36:44.491291872 +0000 UTC m=+0.509234114 container init 5821db7e190181ac053928cc5abc86f3b75feede597a15f6ea9f90a4002d8bee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T05:36:44.650 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:44 smithi053 podman[76116]: 2024-09-13 05:36:44.504262417 +0000 UTC m=+0.522204639 container start 5821db7e190181ac053928cc5abc86f3b75feede597a15f6ea9f90a4002d8bee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:36:44.650 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:44 smithi053 podman[76116]: 2024-09-13 05:36:44.608251921 +0000 UTC m=+0.626194145 container attach 5821db7e190181ac053928cc5abc86f3b75feede597a15f6ea9f90a4002d8bee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:36:44.901 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:44 smithi053 podman[76116]: 2024-09-13 05:36:44.813118512 +0000 UTC m=+0.831060740 container died 5821db7e190181ac053928cc5abc86f3b75feede597a15f6ea9f90a4002d8bee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:36:45.167 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:45 smithi053 ceph-mon[64292]: pgmap v160: 97 pgs: 18 peering, 8 stale+active+clean, 71 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 483 op/s 2024-09-13T05:36:45.167 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:45 smithi053 ceph-mon[64292]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T05:36:45.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 209205 8.84 MB/sec execute 459 sec latency 59.423 ms 2024-09-13T05:36:45.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:45 smithi110 ceph-mon[49387]: pgmap v160: 97 pgs: 18 peering, 8 stale+active+clean, 71 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.7 MiB/s wr, 483 op/s 2024-09-13T05:36:45.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:45 smithi110 ceph-mon[49387]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-13T05:36:45.939 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:45 smithi053 podman[76138]: 2024-09-13 05:36:45.860386552 +0000 UTC m=+1.039030191 container remove 5821db7e190181ac053928cc5abc86f3b75feede597a15f6ea9f90a4002d8bee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T05:36:45.939 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:45 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T05:36:45.939 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:45 smithi053 systemd[1]: Stopped Ceph osd.3 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:36:45.939 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:45 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.3.service: Consumed 4min 8.092s CPU time. 2024-09-13T05:36:46.450 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:46 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 80/1611 objects degraded (4.966%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:46.450 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:46 smithi053 systemd[1]: Starting Ceph osd.3 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:36:46.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 209371 8.83 MB/sec execute 460 sec latency 59.510 ms 2024-09-13T05:36:46.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:46 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 80/1611 objects degraded (4.966%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:46.715 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:46 smithi053 podman[76238]: 2024-09-13 05:36:46.352357913 +0000 UTC m=+0.019242234 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:36:46.715 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:46 smithi053 podman[76238]: 2024-09-13 05:36:46.487464262 +0000 UTC m=+0.154348577 container create 5b41b6db3667db694df1d0fdb4e6378547484cc0194774b95a8b2e045257a836 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD) 2024-09-13T05:36:46.971 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:46 smithi053 podman[76238]: 2024-09-13 05:36:46.830868974 +0000 UTC m=+0.497753283 container init 5b41b6db3667db694df1d0fdb4e6378547484cc0194774b95a8b2e045257a836 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:36:46.971 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:46 smithi053 podman[76238]: 2024-09-13 05:36:46.843094284 +0000 UTC m=+0.509978598 container start 5b41b6db3667db694df1d0fdb4e6378547484cc0194774b95a8b2e045257a836 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:36:46.971 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:46 smithi053 podman[76238]: 2024-09-13 05:36:46.939430574 +0000 UTC m=+0.606314905 container attach 5b41b6db3667db694df1d0fdb4e6378547484cc0194774b95a8b2e045257a836 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-13T05:36:47.235 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:47 smithi053 ceph-mon[64292]: pgmap v161: 97 pgs: 3 active+undersized, 18 peering, 4 stale+active+clean, 10 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 430 op/s; 80/1611 objects degraded (4.966%) 2024-09-13T05:36:47.235 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:47 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:36:47.235 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:47 smithi053 bash[76238]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:36:47.235 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:47 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:36:47.235 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:47 smithi053 bash[76238]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:36:47.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 209534 8.83 MB/sec execute 461 sec latency 68.958 ms 2024-09-13T05:36:47.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:47 smithi110 ceph-mon[49387]: pgmap v161: 97 pgs: 3 active+undersized, 18 peering, 4 stale+active+clean, 10 active+undersized+degraded, 62 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 430 op/s; 80/1611 objects degraded (4.966%) 2024-09-13T05:36:48.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 209744 8.82 MB/sec execute 462 sec latency 62.445 ms 2024-09-13T05:36:49.320 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:49 smithi053 ceph-mon[64292]: pgmap v162: 97 pgs: 7 active+undersized, 18 peering, 16 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 220 op/s; 108/1632 objects degraded (6.618%) 2024-09-13T05:36:49.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 209973 8.81 MB/sec execute 463 sec latency 76.903 ms 2024-09-13T05:36:49.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:49 smithi110 ceph-mon[49387]: pgmap v162: 97 pgs: 7 active+undersized, 18 peering, 16 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 220 op/s; 108/1632 objects degraded (6.618%) 2024-09-13T05:36:50.094 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:49 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T05:36:50.094 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:49 smithi053 bash[76238]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T05:36:50.094 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:49 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T05:36:50.094 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:49 smithi053 bash[76238]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T05:36:50.355 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:50 smithi053 ceph-mon[64292]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-13T05:36:50.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 210187 8.80 MB/sec execute 464 sec latency 88.781 ms 2024-09-13T05:36:50.618 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T05:36:50.618 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 bash[76238]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T05:36:50.618 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 bash[76238]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 bash[76238]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 bash[76238]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate[76253]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 bash[76238]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T05:36:50.619 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:50 smithi053 podman[76238]: 2024-09-13 05:36:50.435490799 +0000 UTC m=+4.102375124 container died 5b41b6db3667db694df1d0fdb4e6378547484cc0194774b95a8b2e045257a836 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T05:36:50.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:50 smithi110 ceph-mon[49387]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-13T05:36:51.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:51 smithi053 ceph-mon[64292]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.2 MiB/s wr, 380 op/s; 201/1650 objects degraded (12.182%) 2024-09-13T05:36:51.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 210392 8.79 MB/sec execute 465 sec latency 44.160 ms 2024-09-13T05:36:51.642 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:51 smithi053 podman[76414]: 2024-09-13 05:36:51.397823149 +0000 UTC m=+0.954240084 container remove 5b41b6db3667db694df1d0fdb4e6378547484cc0194774b95a8b2e045257a836 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T05:36:51.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:51 smithi110 ceph-mon[49387]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.2 MiB/s wr, 380 op/s; 201/1650 objects degraded (12.182%) 2024-09-13T05:36:51.911 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:51 smithi053 podman[76474]: 2024-09-13 05:36:51.545922518 +0000 UTC m=+0.019042457 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:36:51.911 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:51 smithi053 podman[76474]: 2024-09-13 05:36:51.685153823 +0000 UTC m=+0.158273760 container create 91b91cf43436c484136f60e9693e963a126a7b74cf7c2b26722f2a86a51c5400 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:36:52.172 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:52 smithi053 podman[76474]: 2024-09-13 05:36:52.078450893 +0000 UTC m=+0.551570837 container init 91b91cf43436c484136f60e9693e963a126a7b74cf7c2b26722f2a86a51c5400 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T05:36:52.172 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:52 smithi053 podman[76474]: 2024-09-13 05:36:52.086653167 +0000 UTC m=+0.559773098 container start 91b91cf43436c484136f60e9693e963a126a7b74cf7c2b26722f2a86a51c5400 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T05:36:52.172 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:52 smithi053 ceph-osd[76492]: -- 172.21.15.53:0/3866531520 <== mon.0 v2:172.21.15.53:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bf44e0a1e0 con 0x55bf44d98000 2024-09-13T05:36:52.423 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:52 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 201/1656 objects degraded (12.138%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:52.423 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:52.423 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:52 smithi053 bash[76474]: 91b91cf43436c484136f60e9693e963a126a7b74cf7c2b26722f2a86a51c5400 2024-09-13T05:36:52.423 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:52 smithi053 systemd[1]: Started Ceph osd.3 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:36:52.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 210619 8.78 MB/sec execute 466 sec latency 70.185 ms 2024-09-13T05:36:52.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:52 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 201/1656 objects degraded (12.138%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:52.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:52 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:53.434 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:53 smithi053 ceph-mon[64292]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.8 MiB/s wr, 329 op/s; 201/1656 objects degraded (12.138%) 2024-09-13T05:36:53.434 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:53.435 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:36:53.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 210957 8.77 MB/sec execute 467 sec latency 64.203 ms 2024-09-13T05:36:53.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:53 smithi110 ceph-mon[49387]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.8 MiB/s wr, 329 op/s; 201/1656 objects degraded (12.138%) 2024-09-13T05:36:53.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:53.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:36:54.207 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:53 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[76488]: 2024-09-13T05:36:53.984+0000 7f049b2d8640 -1 Falling back to public interface 2024-09-13T05:36:54.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 211338 8.78 MB/sec execute 468 sec latency 91.458 ms 2024-09-13T05:36:55.474 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:55 smithi053 ceph-mon[64292]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.3 MiB/s wr, 294 op/s; 196/1602 objects degraded (12.235%) 2024-09-13T05:36:55.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 211656 8.77 MB/sec execute 469 sec latency 48.166 ms 2024-09-13T05:36:55.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:55 smithi110 ceph-mon[49387]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.3 MiB/s wr, 294 op/s; 196/1602 objects degraded (12.235%) 2024-09-13T05:36:56.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 212022 8.79 MB/sec execute 470 sec latency 73.826 ms 2024-09-13T05:36:57.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 212280 8.78 MB/sec execute 471 sec latency 54.943 ms 2024-09-13T05:36:57.534 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:57 smithi053 ceph-mon[64292]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 287 op/s; 190/1575 objects degraded (12.063%) 2024-09-13T05:36:57.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:57 smithi110 ceph-mon[49387]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 287 op/s; 190/1575 objects degraded (12.063%) 2024-09-13T05:36:58.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 212547 8.79 MB/sec execute 472 sec latency 48.949 ms 2024-09-13T05:36:58.566 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:58 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 190/1575 objects degraded (12.063%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:58.566 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:58.566 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:58 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 190/1575 objects degraded (12.063%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T05:36:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:36:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:36:59.278 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:36:59.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 212775 8.78 MB/sec execute 473 sec latency 44.413 ms 2024-09-13T05:36:59.567 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:36:59 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[76488]: 2024-09-13T05:36:59.441+0000 7f049b2d8640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-13T05:36:59.818 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:36:59 smithi053 ceph-mon[64292]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.4 MiB/s wr, 379 op/s; 183/1542 objects degraded (11.868%) 2024-09-13T05:36:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:36:59 smithi110 ceph-mon[49387]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 3.4 MiB/s wr, 379 op/s; 183/1542 objects degraded (11.868%) 2024-09-13T05:37:00.420 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (6m) 47s ago 16m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (3m) 47s ago 16m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (3m) 3m ago 15m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (6m) 47s ago 16m 44.2M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (10m) 47s ago 10m 38.0M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (10m) 3m ago 10m 16.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (5m) 47s ago 17m 517M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (4m) 3m ago 15m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (4m) 47s ago 17m 50.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (4m) 3m ago 15m 36.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (10m) 47s ago 10m 284M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (10m) 3m ago 10m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:37:00.421 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (6m) 47s ago 16m 9.96M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (5m) 3m ago 15m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (2m) 47s ago 14m 282M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (105s) 47s ago 14m 220M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (56s) 47s ago 13m 12.1M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 starting - - - 1937M 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (13m) 3m ago 13m 527M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (12m) 3m ago 12m 337M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (12m) 3m ago 12m 525M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (11m) 3m ago 11m 431M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:37:00.422 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (4m) 47s ago 15m 53.2M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:37:00.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 212997 8.78 MB/sec execute 474 sec latency 55.794 ms 2024-09-13T05:37:00.837 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:00 smithi053 ceph-mon[64292]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:00.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:00 smithi110 ceph-mon[49387]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:01.057 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 3 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 7 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:37:01.058 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:37:01.102 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:37:00 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[76488]: 2024-09-13T05:37:00.923+0000 7f049b2d8640 -1 osd.3 106 log_to_monitors true 2024-09-13T05:37:01.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 213250 8.78 MB/sec execute 475 sec latency 67.392 ms 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 462 op/s; 195/1602 objects degraded (12.172%) 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: from='osd.3 [v2:172.21.15.53:6826/1985198901,v1:172.21.15.53:6827/1985198901]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T05:37:01.606 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/2546823565' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:01.642 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:37:01.642 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:37:01.642 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:37:01.642 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:37:01.642 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:37:01.642 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:37:01.643 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:37:01.643 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:37:01.643 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:37:01.643 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "10/23 daemons upgraded", 2024-09-13T05:37:01.643 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:37:01.643 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:37:01.643 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:37:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.3 MiB/s wr, 462 op/s; 195/1602 objects degraded (12.172%) 2024-09-13T05:37:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: from='osd.3 [v2:172.21.15.53:6826/1985198901,v1:172.21.15.53:6827/1985198901]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T05:37:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T05:37:01.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:01 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/2546823565' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:02.110 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:37:01 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[76488]: 2024-09-13T05:37:01.881+0000 7f0493082640 -1 osd.3 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:37:02.357 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 195/1602 objects degraded (12.172%), 27 pgs degraded 2024-09-13T05:37:02.357 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T05:37:02.357 INFO:teuthology.orchestra.run.smithi053.stdout: osd.3 (root=default,host=smithi053) is down 2024-09-13T05:37:02.357 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 195/1602 objects degraded (12.172%), 27 pgs degraded 2024-09-13T05:37:02.358 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-13T05:37:02.358 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.9 is active+undersized+degraded, acting [1,2] 2024-09-13T05:37:02.358 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-13T05:37:02.358 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-13T05:37:02.358 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-09-13T05:37:02.358 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-13T05:37:02.358 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-13T05:37:02.359 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-13T05:37:02.360 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-13T05:37:02.360 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-13T05:37:02.360 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-13T05:37:02.360 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-13T05:37:02.360 INFO:teuthology.orchestra.run.smithi053.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-13T05:37:02.360 INFO:teuthology.orchestra.run.smithi053.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-13T05:37:02.360 INFO:teuthology.orchestra.run.smithi053.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-13T05:37:02.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 213607 8.77 MB/sec execute 476 sec latency 38.243 ms 2024-09-13T05:37:02.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:02.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:02.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:02.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T05:37:02.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: osdmap e109: 8 total, 7 up, 8 in 2024-09-13T05:37:02.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: from='osd.3 [v2:172.21.15.53:6826/1985198901,v1:172.21.15.53:6827/1985198901]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:37:02.868 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:37:02.869 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1993232162' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:37:02.869 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:02 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 195/1602 objects degraded (12.172%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: from='client.34326 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:02.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T05:37:02.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: osdmap e109: 8 total, 7 up, 8 in 2024-09-13T05:37:02.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: from='osd.3 [v2:172.21.15.53:6826/1985198901,v1:172.21.15.53:6827/1985198901]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:37:02.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi053", "root=default"]} : dispatch 2024-09-13T05:37:02.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1993232162' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:37:02.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:02 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 195/1602 objects degraded (12.172%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:03.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 213754 8.76 MB/sec execute 477 sec latency 662.725 ms 2024-09-13T05:37:03.681 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:03 smithi053 ceph-mon[64292]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 403 op/s; 207/1644 objects degraded (12.591%) 2024-09-13T05:37:03.682 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:03 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:37:03.682 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:03 smithi053 ceph-mon[64292]: osd.3 [v2:172.21.15.53:6826/1985198901,v1:172.21.15.53:6827/1985198901] boot 2024-09-13T05:37:03.682 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:03 smithi053 ceph-mon[64292]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T05:37:03.682 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T05:37:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:03 smithi110 ceph-mon[49387]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 403 op/s; 207/1644 objects degraded (12.591%) 2024-09-13T05:37:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:03 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:37:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:03 smithi110 ceph-mon[49387]: osd.3 [v2:172.21.15.53:6826/1985198901,v1:172.21.15.53:6827/1985198901] boot 2024-09-13T05:37:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:03 smithi110 ceph-mon[49387]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T05:37:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T05:37:04.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 213849 8.74 MB/sec execute 478 sec latency 1662.751 ms 2024-09-13T05:37:05.039 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:04 smithi053 ceph-mon[64292]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T05:37:05.039 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:04 smithi053 ceph-mon[64292]: pgmap v173: 97 pgs: 10 peering, 9 active+undersized, 22 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.4 MiB/s wr, 402 op/s; 161/1695 objects degraded (9.499%) 2024-09-13T05:37:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:04 smithi110 ceph-mon[49387]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T05:37:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:04 smithi110 ceph-mon[49387]: pgmap v173: 97 pgs: 10 peering, 9 active+undersized, 22 active+undersized+degraded, 56 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.4 MiB/s wr, 402 op/s; 161/1695 objects degraded (9.499%) 2024-09-13T05:37:05.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 214054 8.73 MB/sec execute 479 sec latency 2084.073 ms 2024-09-13T05:37:06.067 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:05 smithi053 ceph-mon[64292]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T05:37:06.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:05 smithi110 ceph-mon[49387]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T05:37:06.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 214214 8.72 MB/sec execute 480 sec latency 640.580 ms 2024-09-13T05:37:07.124 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:06 smithi053 ceph-mon[64292]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T05:37:07.124 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:06 smithi053 ceph-mon[64292]: pgmap v176: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 10 peering, 6 active+undersized, 14 active+undersized+degraded, 64 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 458 op/s; 107/1659 objects degraded (6.450%); 1.4 MiB/s, 16 objects/s recovering 2024-09-13T05:37:07.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:06 smithi110 ceph-mon[49387]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T05:37:07.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:06 smithi110 ceph-mon[49387]: pgmap v176: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 10 peering, 6 active+undersized, 14 active+undersized+degraded, 64 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 458 op/s; 107/1659 objects degraded (6.450%); 1.4 MiB/s, 16 objects/s recovering 2024-09-13T05:37:07.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 214326 8.71 MB/sec execute 481 sec latency 1044.602 ms 2024-09-13T05:37:07.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T05:37:07.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 107/1659 objects degraded (6.450%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:07.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:07.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:07.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:37:07.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:37:07.900 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:07.901 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:07 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: osdmap e114: 8 total, 8 up, 8 in 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 107/1659 objects degraded (6.450%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:07 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:08.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 214600 8.71 MB/sec execute 482 sec latency 370.164 ms 2024-09-13T05:37:08.941 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T05:37:08.941 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: pgmap v179: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 peering, 1 active+undersized+degraded, 82 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.4 MiB/s wr, 199 op/s; 36/1614 objects degraded (2.230%); 464 KiB/s, 24 objects/s recovering 2024-09-13T05:37:08.941 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:37:08.941 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:08.941 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:08.941 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:08.942 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T05:37:08.942 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T05:37:08.942 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:08 smithi053 ceph-mon[64292]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T05:37:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: osdmap e115: 8 total, 8 up, 8 in 2024-09-13T05:37:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: pgmap v179: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 13 peering, 1 active+undersized+degraded, 82 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.4 MiB/s wr, 199 op/s; 36/1614 objects degraded (2.230%); 464 KiB/s, 24 objects/s recovering 2024-09-13T05:37:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:37:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:09.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:09.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T05:37:09.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T05:37:09.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:08 smithi110 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T05:37:09.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 214963 8.71 MB/sec execute 483 sec latency 49.060 ms 2024-09-13T05:37:10.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 215267 8.70 MB/sec execute 484 sec latency 47.604 ms 2024-09-13T05:37:11.261 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:11 smithi053 ceph-mon[64292]: pgmap v180: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 92 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.4 MiB/s wr, 354 op/s; 36/1482 objects degraded (2.429%); 338 KiB/s, 28 objects/s recovering 2024-09-13T05:37:11.475 INFO:teuthology.orchestra.run.smithi053.stdout: 5 215465 8.69 MB/sec execute 485 sec latency 68.893 ms 2024-09-13T05:37:11.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:11 smithi110 ceph-mon[49387]: pgmap v180: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+degraded, 92 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.4 MiB/s wr, 354 op/s; 36/1482 objects degraded (2.429%); 338 KiB/s, 28 objects/s recovering 2024-09-13T05:37:12.289 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:12 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/1482 objects degraded (2.429%), 2 pgs degraded) 2024-09-13T05:37:12.289 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:12 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:37:12.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 215765 8.70 MB/sec execute 486 sec latency 45.659 ms 2024-09-13T05:37:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:12 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 36/1482 objects degraded (2.429%), 2 pgs degraded) 2024-09-13T05:37:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:12 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:37:13.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 216041 8.70 MB/sec execute 487 sec latency 56.203 ms 2024-09-13T05:37:13.577 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:13 smithi053 ceph-mon[64292]: pgmap v181: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 MiB/s wr, 243 op/s; 193 KiB/s, 25 objects/s recovering 2024-09-13T05:37:13.577 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:13.577 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:13 smithi110 ceph-mon[49387]: pgmap v181: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 MiB/s wr, 243 op/s; 193 KiB/s, 25 objects/s recovering 2024-09-13T05:37:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:14.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 216304 8.71 MB/sec execute 488 sec latency 86.719 ms 2024-09-13T05:37:15.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 216558 8.71 MB/sec execute 489 sec latency 81.635 ms 2024-09-13T05:37:15.640 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:15 smithi053 ceph-mon[64292]: pgmap v182: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.5 MiB/s wr, 387 op/s; 150 KiB/s, 20 objects/s recovering 2024-09-13T05:37:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:15 smithi110 ceph-mon[49387]: pgmap v182: 97 pgs: 97 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 3.5 MiB/s wr, 387 op/s; 150 KiB/s, 20 objects/s recovering 2024-09-13T05:37:16.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 216689 8.70 MB/sec execute 490 sec latency 85.608 ms 2024-09-13T05:37:17.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 216844 8.69 MB/sec execute 491 sec latency 69.262 ms 2024-09-13T05:37:17.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:17 smithi053 ceph-mon[64292]: pgmap v183: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.3 MiB/s wr, 337 op/s; 542 KiB/s, 14 objects/s recovering 2024-09-13T05:37:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:17 smithi110 ceph-mon[49387]: pgmap v183: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.3 MiB/s wr, 337 op/s; 542 KiB/s, 14 objects/s recovering 2024-09-13T05:37:18.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 217094 8.68 MB/sec execute 492 sec latency 50.248 ms 2024-09-13T05:37:18.740 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:19.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 217288 8.67 MB/sec execute 493 sec latency 71.675 ms 2024-09-13T05:37:19.755 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:19 smithi053 ceph-mon[64292]: pgmap v184: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.3 MiB/s wr, 316 op/s; 490 KiB/s, 12 objects/s recovering 2024-09-13T05:37:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:19 smithi110 ceph-mon[49387]: pgmap v184: 97 pgs: 97 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.3 MiB/s wr, 316 op/s; 490 KiB/s, 12 objects/s recovering 2024-09-13T05:37:20.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 217496 8.66 MB/sec execute 494 sec latency 90.352 ms 2024-09-13T05:37:21.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 217701 8.65 MB/sec execute 495 sec latency 54.975 ms 2024-09-13T05:37:21.759 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:21 smithi053 ceph-mon[64292]: pgmap v185: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 388 op/s; 416 KiB/s, 10 objects/s recovering 2024-09-13T05:37:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:21 smithi110 ceph-mon[49387]: pgmap v185: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.0 MiB/s wr, 388 op/s; 416 KiB/s, 10 objects/s recovering 2024-09-13T05:37:22.476 INFO:teuthology.orchestra.run.smithi053.stdout: 5 218003 8.65 MB/sec execute 496 sec latency 56.250 ms 2024-09-13T05:37:23.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 218492 8.65 MB/sec execute 497 sec latency 39.410 ms 2024-09-13T05:37:23.847 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:23 smithi053 ceph-mon[64292]: pgmap v186: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 325 op/s; 416 KiB/s, 6 objects/s recovering 2024-09-13T05:37:23.848 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:23 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T05:37:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:23 smithi110 ceph-mon[49387]: pgmap v186: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 325 op/s; 416 KiB/s, 6 objects/s recovering 2024-09-13T05:37:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:23 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T05:37:24.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 218793 8.65 MB/sec execute 498 sec latency 80.057 ms 2024-09-13T05:37:24.882 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:24 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T05:37:24.882 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:24 smithi053 ceph-mon[64292]: Upgrade: osd.4 is safe to restart 2024-09-13T05:37:24.946 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:24 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T05:37:24.947 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:24 smithi110 ceph-mon[49387]: Upgrade: osd.4 is safe to restart 2024-09-13T05:37:25.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 219030 8.64 MB/sec execute 499 sec latency 44.341 ms 2024-09-13T05:37:25.914 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:25 smithi053 ceph-mon[64292]: pgmap v187: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 433 op/s 2024-09-13T05:37:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:25 smithi110 ceph-mon[49387]: pgmap v187: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 433 op/s 2024-09-13T05:37:26.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 219314 8.65 MB/sec execute 500 sec latency 61.842 ms 2024-09-13T05:37:27.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:27 smithi053 ceph-mon[64292]: pgmap v188: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.2 MiB/s wr, 317 op/s 2024-09-13T05:37:27.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:27 smithi053 ceph-mon[64292]: Upgrade: Updating osd.4 2024-09-13T05:37:27.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:27.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T05:37:27.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:37:27.470 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:27 smithi053 ceph-mon[64292]: Deploying daemon osd.4 on smithi110 2024-09-13T05:37:27.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 219555 8.65 MB/sec execute 501 sec latency 45.878 ms 2024-09-13T05:37:27.724 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:27 smithi110 ceph-mon[49387]: pgmap v188: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.2 MiB/s wr, 317 op/s 2024-09-13T05:37:27.724 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:27 smithi110 ceph-mon[49387]: Upgrade: Updating osd.4 2024-09-13T05:37:27.724 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:27.725 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T05:37:27.725 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:37:27.725 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:27 smithi110 ceph-mon[49387]: Deploying daemon osd.4 on smithi110 2024-09-13T05:37:28.284 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:28 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:28.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 219825 8.65 MB/sec execute 502 sec latency 45.282 ms 2024-09-13T05:37:28.503 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:28 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:29.267 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:29 smithi053 ceph-mon[64292]: pgmap v189: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 326 op/s 2024-09-13T05:37:29.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 220173 8.66 MB/sec execute 503 sec latency 53.812 ms 2024-09-13T05:37:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:29 smithi110 ceph-mon[49387]: pgmap v189: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 326 op/s 2024-09-13T05:37:29.944 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:29 smithi110 systemd[1]: Stopping Ceph osd.4 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:37:30.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:29 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[30044]: 2024-09-13T05:37:29.942+0000 7fe89025c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:37:30.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:29 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[30044]: 2024-09-13T05:37:29.942+0000 7fe89025c700 -1 osd.4 115 *** Got signal Terminated *** 2024-09-13T05:37:30.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:29 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[30044]: 2024-09-13T05:37:29.942+0000 7fe89025c700 -1 osd.4 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:37:30.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 220369 8.65 MB/sec execute 504 sec latency 219.557 ms 2024-09-13T05:37:30.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:30 smithi053 ceph-mon[64292]: osd.4 marked itself down and dead 2024-09-13T05:37:30.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:30 smithi110 ceph-mon[49387]: osd.4 marked itself down and dead 2024-09-13T05:37:31.143 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:30 smithi110 podman[52793]: 2024-09-13 05:37:30.879668829 +0000 UTC m=+1.060977052 container died 2b1bd7b36101d11e23ea43314d0d8f51690aba77beec2f1785bd9e6656d200c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, ceph=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, distribution-scope=public, GIT_CLEAN=True, release=754, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, RELEASE=HEAD, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T05:37:31.394 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:31 smithi110 ceph-mon[49387]: pgmap v190: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 446 op/s 2024-09-13T05:37:31.394 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:31 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:37:31.394 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:31 smithi110 ceph-mon[49387]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T05:37:31.395 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:31 smithi110 podman[52793]: 2024-09-13 05:37:31.217280729 +0000 UTC m=+1.398588950 container cleanup 2b1bd7b36101d11e23ea43314d0d8f51690aba77beec2f1785bd9e6656d200c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, vcs-type=git, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, GIT_CLEAN=True, name=centos-stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, ceph=True, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0) 2024-09-13T05:37:31.395 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:31 smithi110 bash[52793]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4 2024-09-13T05:37:31.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 220407 8.64 MB/sec execute 505 sec latency 1041.814 ms 2024-09-13T05:37:31.526 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:31 smithi053 ceph-mon[64292]: pgmap v190: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 446 op/s 2024-09-13T05:37:31.526 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:31 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:37:31.526 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:31 smithi053 ceph-mon[64292]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T05:37:31.744 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:31 smithi110 podman[52811]: 2024-09-13 05:37:31.394022237 +0000 UTC m=+0.512376810 container remove 2b1bd7b36101d11e23ea43314d0d8f51690aba77beec2f1785bd9e6656d200c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, vcs-type=git, version=8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, name=centos-stream) 2024-09-13T05:37:32.106 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:31 smithi110 podman[52897]: 2024-09-13 05:37:31.754689919 +0000 UTC m=+0.021099055 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:37:32.106 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:31 smithi110 podman[52897]: 2024-09-13 05:37:31.870216046 +0000 UTC m=+0.136625158 container create ba40eb9aed3a8809e77706f4d428bf3d03d237b7d67f2f1a814203563a984909 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:37:32.106 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 podman[52897]: 2024-09-13 05:37:32.106221509 +0000 UTC m=+0.372630618 container init ba40eb9aed3a8809e77706f4d428bf3d03d237b7d67f2f1a814203563a984909 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-13T05:37:32.296 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:32 smithi053 ceph-mon[64292]: osdmap e117: 8 total, 7 up, 8 in 2024-09-13T05:37:32.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 220690 8.63 MB/sec execute 506 sec latency 44.927 ms 2024-09-13T05:37:32.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:32 smithi110 ceph-mon[49387]: osdmap e117: 8 total, 7 up, 8 in 2024-09-13T05:37:32.494 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 podman[52897]: 2024-09-13 05:37:32.109975758 +0000 UTC m=+0.376384877 container start ba40eb9aed3a8809e77706f4d428bf3d03d237b7d67f2f1a814203563a984909 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:37:32.495 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 podman[52897]: 2024-09-13 05:37:32.206109671 +0000 UTC m=+0.472518796 container attach ba40eb9aed3a8809e77706f4d428bf3d03d237b7d67f2f1a814203563a984909 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:37:32.495 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 podman[52897]: 2024-09-13 05:37:32.314526002 +0000 UTC m=+0.580935112 container died ba40eb9aed3a8809e77706f4d428bf3d03d237b7d67f2f1a814203563a984909 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2) 2024-09-13T05:37:32.876 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:37:33.245 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 podman[52920]: 2024-09-13 05:37:32.96225103 +0000 UTC m=+0.641567138 container remove ba40eb9aed3a8809e77706f4d428bf3d03d237b7d67f2f1a814203563a984909 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731) 2024-09-13T05:37:33.245 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T05:37:33.245 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 systemd[1]: Stopped Ceph osd.4 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:37:33.245 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:32 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 48.015s CPU time. 2024-09-13T05:37:33.245 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 systemd[1]: Starting Ceph osd.4 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:37:33.333 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:33 smithi053 ceph-mon[64292]: pgmap v193: 97 pgs: 4 active+undersized, 1 activating+undersized+degraded, 5 activating+undersized, 8 peering, 7 stale+active+clean, 1 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.8 MiB/s wr, 342 op/s; 54/1668 objects degraded (3.237%) 2024-09-13T05:37:33.333 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:33 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 54/1668 objects degraded (3.237%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:33.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 220945 8.62 MB/sec execute 507 sec latency 86.898 ms 2024-09-13T05:37:33.537 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:33 smithi110 ceph-mon[49387]: pgmap v193: 97 pgs: 4 active+undersized, 1 activating+undersized+degraded, 5 activating+undersized, 8 peering, 7 stale+active+clean, 1 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.8 MiB/s wr, 342 op/s; 54/1668 objects degraded (3.237%) 2024-09-13T05:37:33.537 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:33 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 54/1668 objects degraded (3.237%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:33.538 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 podman[53020]: 2024-09-13 05:37:33.312286959 +0000 UTC m=+0.016478103 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:37:33.538 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 podman[53020]: 2024-09-13 05:37:33.430391591 +0000 UTC m=+0.134582726 container create 4377159cf95f0588114994b5b09e8403f6616ab640dc12536cd66353962c014b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T05:37:33.817 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:37:33.817 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (7m) 33s ago 17m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:37:33.817 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (3m) 33s ago 16m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:37:33.817 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (3m) 3m ago 16m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:37:33.817 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (6m) 33s ago 16m 44.3M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (11m) 33s ago 11m 38.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (10m) 3m ago 10m 16.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (6m) 33s ago 18m 518M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (5m) 3m ago 16m 453M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (5m) 33s ago 18m 52.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (4m) 3m ago 16m 36.4M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (10m) 33s ago 10m 279M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (10m) 3m ago 10m 90.2M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (6m) 33s ago 16m 10.2M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (6m) 3m ago 16m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (3m) 33s ago 15m 307M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (2m) 33s ago 15m 248M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:37:33.818 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (90s) 33s ago 14m 195M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:37:33.819 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (42s) 33s ago 14m 14.2M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:37:33.819 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (13m) 3m ago 13m 527M 2704M 17.2.0 e1d6a67b021e 2b1bd7b36101 2024-09-13T05:37:33.819 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (13m) 3m ago 13m 337M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:37:33.819 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (12m) 3m ago 12m 525M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:37:33.819 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (12m) 3m ago 12m 431M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:37:33.819 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (5m) 33s ago 15m 54.0M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:37:33.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 podman[53020]: 2024-09-13 05:37:33.658533728 +0000 UTC m=+0.362724873 container init 4377159cf95f0588114994b5b09e8403f6616ab640dc12536cd66353962c014b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T05:37:33.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 podman[53020]: 2024-09-13 05:37:33.661949572 +0000 UTC m=+0.366140712 container start 4377159cf95f0588114994b5b09e8403f6616ab640dc12536cd66353962c014b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T05:37:33.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 podman[53020]: 2024-09-13 05:37:33.76708068 +0000 UTC m=+0.471271820 container attach 4377159cf95f0588114994b5b09e8403f6616ab640dc12536cd66353962c014b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T05:37:33.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:37:33.995 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 bash[53020]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:37:33.995 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:37:33.995 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:33 smithi110 bash[53020]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:37:34.368 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:34 smithi053 ceph-mon[64292]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:34.368 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:34 smithi053 ceph-mon[64292]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:34.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 221204 8.61 MB/sec execute 508 sec latency 51.871 ms 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 4 2024-09-13T05:37:34.478 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 8 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:37:34.479 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:37:34.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:34 smithi110 ceph-mon[49387]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:34.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:34 smithi110 ceph-mon[49387]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:34.957 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:37:34.957 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "10/23 daemons upgraded", 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:37:34.958 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:37:35.329 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:35 smithi053 ceph-mon[64292]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:35.330 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:35 smithi053 ceph-mon[64292]: pgmap v194: 97 pgs: 8 active+undersized, 1 activating+undersized+degraded, 5 activating+undersized, 8 peering, 2 stale+active+clean, 16 active+undersized+degraded, 57 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.4 MiB/s wr, 395 op/s; 184/1686 objects degraded (10.913%) 2024-09-13T05:37:35.330 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:35 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1684334852' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:35.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 221523 8.61 MB/sec execute 509 sec latency 78.008 ms 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 184/1686 objects degraded (10.913%), 17 pgs degraded 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: osd.4 (root=default,host=smithi110) is down 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 184/1686 objects degraded (10.913%), 17 pgs degraded 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T05:37:35.624 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.1a is activating+undersized+degraded, acting [1,2] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T05:37:35.625 INFO:teuthology.orchestra.run.smithi053.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-13T05:37:35.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:35 smithi110 ceph-mon[49387]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:35.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:35 smithi110 ceph-mon[49387]: pgmap v194: 97 pgs: 8 active+undersized, 1 activating+undersized+degraded, 5 activating+undersized, 8 peering, 2 stale+active+clean, 16 active+undersized+degraded, 57 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.4 MiB/s wr, 395 op/s; 184/1686 objects degraded (10.913%) 2024-09-13T05:37:35.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:35 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1684334852' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:36.214 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:35 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T05:37:36.214 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:35 smithi110 bash[53020]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T05:37:36.214 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:35 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T05:37:36.214 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:35 smithi110 bash[53020]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T05:37:36.214 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T05:37:36.214 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 bash[53020]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T05:37:36.420 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:36 smithi053 ceph-mon[64292]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:36.420 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:36 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/429651619' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:37:36.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 221939 8.61 MB/sec execute 510 sec latency 37.090 ms 2024-09-13T05:37:36.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:36 smithi110 ceph-mon[49387]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:37:36.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:36 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/429651619' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:37:36.494 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T05:37:36.494 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 bash[53020]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T05:37:36.494 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T05:37:36.494 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 bash[53020]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T05:37:36.495 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T05:37:36.495 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 bash[53020]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T05:37:36.495 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate[53035]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T05:37:36.495 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 bash[53020]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T05:37:36.495 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 podman[53020]: 2024-09-13 05:37:36.24413056 +0000 UTC m=+2.948321702 container died 4377159cf95f0588114994b5b09e8403f6616ab640dc12536cd66353962c014b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T05:37:37.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:36 smithi110 podman[53210]: 2024-09-13 05:37:36.922430634 +0000 UTC m=+0.668188957 container remove 4377159cf95f0588114994b5b09e8403f6616ab640dc12536cd66353962c014b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:37:37.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:37 smithi110 podman[53269]: 2024-09-13 05:37:37.034821643 +0000 UTC m=+0.016423011 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:37:37.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:37 smithi110 podman[53269]: 2024-09-13 05:37:37.148531936 +0000 UTC m=+0.130133307 container create c09b6fed14c0329944856266ed78967b07fccfd1271962721728023e75276787 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:37:37.432 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:37 smithi053 ceph-mon[64292]: pgmap v195: 97 pgs: 11 active+undersized, 1 activating+undersized+degraded, 5 activating+undersized, 24 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 400 op/s; 254/1629 objects degraded (15.592%) 2024-09-13T05:37:37.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 222357 8.62 MB/sec execute 511 sec latency 43.446 ms 2024-09-13T05:37:37.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:37 smithi110 ceph-mon[49387]: pgmap v195: 97 pgs: 11 active+undersized, 1 activating+undersized+degraded, 5 activating+undersized, 24 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.4 MiB/s wr, 400 op/s; 254/1629 objects degraded (15.592%) 2024-09-13T05:37:37.744 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:37 smithi110 podman[53269]: 2024-09-13 05:37:37.377430524 +0000 UTC m=+0.359031902 container init c09b6fed14c0329944856266ed78967b07fccfd1271962721728023e75276787 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True) 2024-09-13T05:37:37.745 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:37 smithi110 podman[53269]: 2024-09-13 05:37:37.381298412 +0000 UTC m=+0.362899790 container start c09b6fed14c0329944856266ed78967b07fccfd1271962721728023e75276787 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0) 2024-09-13T05:37:37.745 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:37 smithi110 ceph-osd[53288]: -- 172.21.15.110:0/2163272988 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eaad4081e0 con 0x55eaac5a3c00 2024-09-13T05:37:37.745 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:37 smithi110 bash[53269]: c09b6fed14c0329944856266ed78967b07fccfd1271962721728023e75276787 2024-09-13T05:37:37.745 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:37 smithi110 systemd[1]: Started Ceph osd.4 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:37:38.431 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:38 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 254/1629 objects degraded (15.592%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:38.431 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:38.431 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:38.431 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:37:38.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 222674 8.61 MB/sec execute 512 sec latency 52.685 ms 2024-09-13T05:37:38.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:38 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 254/1629 objects degraded (15.592%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:38.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:38.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:38.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:37:39.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 222917 8.62 MB/sec execute 513 sec latency 84.726 ms 2024-09-13T05:37:39.584 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:39 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:37:39.277+0000 7f0707d63640 -1 Falling back to public interface 2024-09-13T05:37:39.763 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:39 smithi053 ceph-mon[64292]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 292 op/s; 243/1608 objects degraded (15.112%) 2024-09-13T05:37:39.897 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:39 smithi110 ceph-mon[49387]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 292 op/s; 243/1608 objects degraded (15.112%) 2024-09-13T05:37:40.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 223251 8.63 MB/sec execute 514 sec latency 93.208 ms 2024-09-13T05:37:41.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 223541 8.63 MB/sec execute 515 sec latency 75.634 ms 2024-09-13T05:37:41.812 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:41 smithi053 ceph-mon[64292]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 370 op/s; 238/1593 objects degraded (14.940%) 2024-09-13T05:37:41.813 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:41.813 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:41 smithi110 ceph-mon[49387]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.9 MiB/s wr, 370 op/s; 238/1593 objects degraded (14.940%) 2024-09-13T05:37:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:42.477 INFO:teuthology.orchestra.run.smithi053.stdout: 5 223772 8.63 MB/sec execute 516 sec latency 52.581 ms 2024-09-13T05:37:43.092 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:43.093 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:43.093 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:42 smithi053 ceph-mon[64292]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 470 op/s; 248/1650 objects degraded (15.030%) 2024-09-13T05:37:43.093 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:42 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 238/1593 objects degraded (14.940%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:43.093 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:43.093 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:42 smithi110 ceph-mon[49387]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 470 op/s; 248/1650 objects degraded (15.030%) 2024-09-13T05:37:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:42 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 238/1593 objects degraded (14.940%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:43.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 223940 8.62 MB/sec execute 517 sec latency 74.761 ms 2024-09-13T05:37:44.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 224183 8.62 MB/sec execute 518 sec latency 44.629 ms 2024-09-13T05:37:45.140 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:45 smithi110 ceph-mon[49387]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 446 op/s; 258/1677 objects degraded (15.385%) 2024-09-13T05:37:45.190 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:45 smithi053 ceph-mon[64292]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.5 MiB/s wr, 446 op/s; 258/1677 objects degraded (15.385%) 2024-09-13T05:37:45.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 224399 8.60 MB/sec execute 519 sec latency 51.988 ms 2024-09-13T05:37:45.494 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:37:45.139+0000 7f0707d63640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-13T05:37:46.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 224655 8.59 MB/sec execute 520 sec latency 88.107 ms 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:37:46.507 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:46.508 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:46.508 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:46.508 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:46 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T05:37:46.744 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:37:46.393+0000 7f0707d63640 -1 osd.4 115 log_to_monitors true 2024-09-13T05:37:46.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:37:46.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:37:46.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:46.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:37:46.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:46.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:46.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:37:46.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:46 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T05:37:47.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 225009 8.60 MB/sec execute 521 sec latency 40.902 ms 2024-09-13T05:37:47.528 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:47 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T05:37:47.528 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:47 smithi053 ceph-mon[64292]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T05:37:47.528 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:47 smithi053 ceph-mon[64292]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.1 MiB/s wr, 407 op/s; 257/1677 objects degraded (15.325%) 2024-09-13T05:37:47.528 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:47 smithi053 ceph-mon[64292]: from='osd.4 [v2:172.21.15.110:6800/3874736949,v1:172.21.15.110:6801/3874736949]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T05:37:47.528 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:47 smithi053 ceph-mon[64292]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T05:37:47.744 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:37:47 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:37:47.376+0000 7f06ffb0d640 -1 osd.4 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:37:47.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:47 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T05:37:47.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:47 smithi110 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-13T05:37:47.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:47 smithi110 ceph-mon[49387]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.1 MiB/s wr, 407 op/s; 257/1677 objects degraded (15.325%) 2024-09-13T05:37:47.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:47 smithi110 ceph-mon[49387]: from='osd.4 [v2:172.21.15.110:6800/3874736949,v1:172.21.15.110:6801/3874736949]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T05:37:47.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:47 smithi110 ceph-mon[49387]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T05:37:48.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 225382 8.60 MB/sec execute 522 sec latency 110.578 ms 2024-09-13T05:37:48.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:48 smithi110 ceph-mon[49387]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T05:37:48.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:48 smithi110 ceph-mon[49387]: from='osd.4 [v2:172.21.15.110:6800/3874736949,v1:172.21.15.110:6801/3874736949]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:37:48.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:48 smithi110 ceph-mon[49387]: osdmap e118: 8 total, 7 up, 8 in 2024-09-13T05:37:48.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:48 smithi110 ceph-mon[49387]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:37:48.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:48 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 257/1677 objects degraded (15.325%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:48.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:48 smithi053 ceph-mon[64292]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T05:37:48.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:48 smithi053 ceph-mon[64292]: from='osd.4 [v2:172.21.15.110:6800/3874736949,v1:172.21.15.110:6801/3874736949]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:37:48.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:48 smithi053 ceph-mon[64292]: osdmap e118: 8 total, 7 up, 8 in 2024-09-13T05:37:48.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:48 smithi053 ceph-mon[64292]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:37:48.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:48 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 257/1677 objects degraded (15.325%), 28 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:49.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 225391 8.58 MB/sec execute 523 sec latency 1110.611 ms 2024-09-13T05:37:49.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:49 smithi053 ceph-mon[64292]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 485 op/s; 254/1626 objects degraded (15.621%) 2024-09-13T05:37:49.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:49 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:37:49.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:49 smithi053 ceph-mon[64292]: osd.4 [v2:172.21.15.110:6800/3874736949,v1:172.21.15.110:6801/3874736949] boot 2024-09-13T05:37:49.616 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:49 smithi053 ceph-mon[64292]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T05:37:49.617 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:49 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T05:37:49.645 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:49 smithi110 ceph-mon[49387]: pgmap v202: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 352 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 485 op/s; 254/1626 objects degraded (15.621%) 2024-09-13T05:37:49.646 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:49 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:37:49.646 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:49 smithi110 ceph-mon[49387]: osd.4 [v2:172.21.15.110:6800/3874736949,v1:172.21.15.110:6801/3874736949] boot 2024-09-13T05:37:49.646 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:49 smithi110 ceph-mon[49387]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T05:37:49.646 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:49 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T05:37:50.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 225466 8.57 MB/sec execute 524 sec latency 2110.686 ms 2024-09-13T05:37:50.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:50 smithi110 ceph-mon[49387]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T05:37:50.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:50 smithi110 ceph-mon[49387]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T05:37:50.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:50 smithi053 ceph-mon[64292]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T05:37:50.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:50 smithi053 ceph-mon[64292]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T05:37:51.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 225622 8.56 MB/sec execute 525 sec latency 2599.857 ms 2024-09-13T05:37:51.488 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:51 smithi053 ceph-mon[64292]: pgmap v205: 97 pgs: 1 remapped+peering, 12 active+undersized, 27 active+undersized+degraded, 57 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 2.8 MiB/s wr, 322 op/s; 224/1593 objects degraded (14.062%) 2024-09-13T05:37:51.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:51 smithi110 ceph-mon[49387]: pgmap v205: 97 pgs: 1 remapped+peering, 12 active+undersized, 27 active+undersized+degraded, 57 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 2.8 MiB/s wr, 322 op/s; 224/1593 objects degraded (14.062%) 2024-09-13T05:37:52.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 226020 8.55 MB/sec execute 526 sec latency 28.690 ms 2024-09-13T05:37:52.503 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:52 smithi053 ceph-mon[64292]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T05:37:52.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:52 smithi110 ceph-mon[49387]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T05:37:53.437 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:53 smithi053 ceph-mon[64292]: pgmap v208: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 9 active+undersized, 17 active+undersized+degraded, 64 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.1 MiB/s wr, 362 op/s; 223/1539 objects degraded (14.490%); 464 KiB/s, 47 keys/s, 7 objects/s recovering 2024-09-13T05:37:53.437 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:53 smithi053 ceph-mon[64292]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T05:37:53.437 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:53 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 223/1539 objects degraded (14.490%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:53.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 226281 8.56 MB/sec execute 527 sec latency 925.038 ms 2024-09-13T05:37:53.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:53 smithi110 ceph-mon[49387]: pgmap v208: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 9 active+undersized, 17 active+undersized+degraded, 64 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.1 MiB/s wr, 362 op/s; 223/1539 objects degraded (14.490%); 464 KiB/s, 47 keys/s, 7 objects/s recovering 2024-09-13T05:37:53.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:53 smithi110 ceph-mon[49387]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T05:37:53.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:53 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 223/1539 objects degraded (14.490%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:54.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 226608 8.57 MB/sec execute 528 sec latency 900.970 ms 2024-09-13T05:37:54.570 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:54 smithi053 ceph-mon[64292]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T05:37:54.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:54 smithi110 ceph-mon[49387]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T05:37:55.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 227030 8.58 MB/sec execute 529 sec latency 1016.049 ms 2024-09-13T05:37:55.580 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:55 smithi053 ceph-mon[64292]: pgmap v211: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 86 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.3 MiB/s wr, 206 op/s; 154/1569 objects degraded (9.815%); 2.5 MiB/s, 47 keys/s, 41 objects/s recovering 2024-09-13T05:37:55.581 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:55 smithi053 ceph-mon[64292]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T05:37:55.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:55 smithi110 ceph-mon[49387]: pgmap v211: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 86 active+clean; 342 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.3 MiB/s wr, 206 op/s; 154/1569 objects degraded (9.815%); 2.5 MiB/s, 47 keys/s, 41 objects/s recovering 2024-09-13T05:37:55.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:55 smithi110 ceph-mon[49387]: osdmap e125: 8 total, 8 up, 8 in 2024-09-13T05:37:56.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 227433 8.58 MB/sec execute 530 sec latency 160.630 ms 2024-09-13T05:37:56.589 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:56 smithi053 ceph-mon[64292]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T05:37:56.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:56 smithi110 ceph-mon[49387]: osdmap e126: 8 total, 8 up, 8 in 2024-09-13T05:37:57.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 227781 8.59 MB/sec execute 531 sec latency 61.274 ms 2024-09-13T05:37:57.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:57 smithi053 ceph-mon[64292]: pgmap v214: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 89 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.9 MiB/s wr, 332 op/s; 92/1569 objects degraded (5.864%); 2.2 MiB/s, 49 objects/s recovering 2024-09-13T05:37:57.594 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:57 smithi053 ceph-mon[64292]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T05:37:57.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:57 smithi110 ceph-mon[49387]: pgmap v214: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 89 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.9 MiB/s wr, 332 op/s; 92/1569 objects degraded (5.864%); 2.2 MiB/s, 49 objects/s recovering 2024-09-13T05:37:57.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:57 smithi110 ceph-mon[49387]: osdmap e127: 8 total, 8 up, 8 in 2024-09-13T05:37:58.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 228030 8.58 MB/sec execute 532 sec latency 52.370 ms 2024-09-13T05:37:58.618 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:58 smithi053 ceph-mon[64292]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T05:37:58.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:58 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 92/1569 objects degraded (5.864%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:58.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:58.619 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:58 smithi110 ceph-mon[49387]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T05:37:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:58 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 92/1569 objects degraded (5.864%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T05:37:58.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:37:58.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:37:59.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 228353 8.58 MB/sec execute 533 sec latency 45.615 ms 2024-09-13T05:37:59.654 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:37:59 smithi053 ceph-mon[64292]: pgmap v217: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 355 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 5.6 MiB/s wr, 581 op/s; 57/1689 objects degraded (3.375%); 315 KiB/s, 29 objects/s recovering 2024-09-13T05:37:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:37:59 smithi110 ceph-mon[49387]: pgmap v217: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 355 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 5.6 MiB/s wr, 581 op/s; 57/1689 objects degraded (3.375%); 315 KiB/s, 29 objects/s recovering 2024-09-13T05:38:00.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 228652 8.57 MB/sec execute 534 sec latency 58.277 ms 2024-09-13T05:38:00.741 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:00 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/1689 objects degraded (3.375%), 3 pgs degraded) 2024-09-13T05:38:00.742 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:00 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:38:00.917 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:00 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 57/1689 objects degraded (3.375%), 3 pgs degraded) 2024-09-13T05:38:00.917 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:00 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:38:01.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 228989 8.56 MB/sec execute 535 sec latency 55.705 ms 2024-09-13T05:38:01.780 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:01 smithi053 ceph-mon[64292]: pgmap v218: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 466 op/s; 222 KiB/s, 32 objects/s recovering 2024-09-13T05:38:01.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:01 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T05:38:01.914 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:01 smithi110 ceph-mon[49387]: pgmap v218: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 466 op/s; 222 KiB/s, 32 objects/s recovering 2024-09-13T05:38:01.914 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:01 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T05:38:02.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 229170 8.56 MB/sec execute 536 sec latency 71.338 ms 2024-09-13T05:38:02.799 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:02 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T05:38:02.800 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:02 smithi053 ceph-mon[64292]: Upgrade: osd.5 is safe to restart 2024-09-13T05:38:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:02 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T05:38:02.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:02 smithi110 ceph-mon[49387]: Upgrade: osd.5 is safe to restart 2024-09-13T05:38:03.478 INFO:teuthology.orchestra.run.smithi053.stdout: 5 229459 8.56 MB/sec execute 537 sec latency 78.972 ms 2024-09-13T05:38:03.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:03 smithi053 ceph-mon[64292]: pgmap v219: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 485 op/s; 189 KiB/s, 27 objects/s recovering 2024-09-13T05:38:03.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:03.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T05:38:03.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:38:03.937 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:03 smithi110 ceph-mon[49387]: pgmap v219: 97 pgs: 97 active+clean; 352 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.6 MiB/s wr, 485 op/s; 189 KiB/s, 27 objects/s recovering 2024-09-13T05:38:03.937 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:03.937 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T05:38:03.937 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:38:04.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 229775 8.57 MB/sec execute 538 sec latency 58.018 ms 2024-09-13T05:38:04.844 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:04 smithi053 ceph-mon[64292]: Upgrade: Updating osd.5 2024-09-13T05:38:04.845 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:04 smithi053 ceph-mon[64292]: Deploying daemon osd.5 on smithi110 2024-09-13T05:38:04.882 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:04 smithi110 ceph-mon[49387]: Upgrade: Updating osd.5 2024-09-13T05:38:04.882 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:04 smithi110 ceph-mon[49387]: Deploying daemon osd.5 on smithi110 2024-09-13T05:38:05.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 230058 8.56 MB/sec execute 539 sec latency 59.109 ms 2024-09-13T05:38:05.884 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:05 smithi053 ceph-mon[64292]: pgmap v220: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 548 op/s; 79 KiB/s, 14 objects/s recovering 2024-09-13T05:38:05.925 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:05 smithi110 ceph-mon[49387]: pgmap v220: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 548 op/s; 79 KiB/s, 14 objects/s recovering 2024-09-13T05:38:06.098 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:38:06.456 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:06 smithi110 systemd[1]: Stopping Ceph osd.5 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:38:06.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 230335 8.57 MB/sec execute 540 sec latency 82.081 ms 2024-09-13T05:38:06.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:06 smithi110 ceph-mon[49387]: osd.5 marked itself down and dead 2024-09-13T05:38:06.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:06 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[32986]: 2024-09-13T05:38:06.503+0000 7fd78e902700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:38:06.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:06 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[32986]: 2024-09-13T05:38:06.503+0000 7fd78e902700 -1 osd.5 128 *** Got signal Terminated *** 2024-09-13T05:38:06.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:06 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[32986]: 2024-09-13T05:38:06.503+0000 7fd78e902700 -1 osd.5 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:38:06.912 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:06 smithi053 ceph-mon[64292]: osd.5 marked itself down and dead 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (7m) 66s ago 17m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (4m) 66s ago 17m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (4m) 25s ago 16m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (7m) 66s ago 17m 44.3M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (11m) 66s ago 11m 38.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (11m) 25s ago 11m 16.7M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (6m) 66s ago 18m 518M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (5m) 25s ago 16m 455M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (5m) 66s ago 18m 52.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:38:06.971 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (5m) 25s ago 16m 40.2M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (11m) 66s ago 11m 279M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (11m) 25s ago 11m 90.7M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (7m) 66s ago 17m 10.2M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (6m) 25s ago 16m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (3m) 66s ago 16m 307M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (2m) 66s ago 15m 248M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (2m) 66s ago 15m 195M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (75s) 66s ago 14m 14.2M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (29s) 25s ago 14m 12.1M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (13m) 25s ago 13m 524M 2704M 17.2.0 e1d6a67b021e 4c65de07497b 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (13m) 25s ago 13m 443M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (13m) 25s ago 13m 701M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:38:06.972 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (5m) 66s ago 16m 54.0M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:07.479 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 5 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T05:38:07.480 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 9 2024-09-13T05:38:07.481 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:38:07.481 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:38:07.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 230467 8.56 MB/sec execute 541 sec latency 784.344 ms 2024-09-13T05:38:07.666 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:07 smithi110 podman[54835]: 2024-09-13 05:38:07.336797077 +0000 UTC m=+0.957120783 container died 4c65de07497b5478247300a584597a847f193db44a54145757a3ffd1772a6b32 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, name=centos-stream, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, version=8) 2024-09-13T05:38:07.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:07 smithi053 ceph-mon[64292]: pgmap v221: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 372 op/s; 65 KiB/s, 11 objects/s recovering 2024-09-13T05:38:07.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:07 smithi053 ceph-mon[64292]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:07.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:07 smithi053 ceph-mon[64292]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:07.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:07 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:38:07.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:07 smithi053 ceph-mon[64292]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T05:38:07.733 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:07 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1761579659' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:07.902 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:38:07.902 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:38:07.902 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:38:07.902 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "11/23 daemons upgraded", 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:38:07.903 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:38:07.971 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:07 smithi110 ceph-mon[49387]: pgmap v221: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 372 op/s; 65 KiB/s, 11 objects/s recovering 2024-09-13T05:38:07.971 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:07 smithi110 ceph-mon[49387]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:07.972 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:07 smithi110 ceph-mon[49387]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:07.972 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:07 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:38:07.972 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:07 smithi110 ceph-mon[49387]: osdmap e129: 8 total, 7 up, 8 in 2024-09-13T05:38:07.972 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:07 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1761579659' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:08.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:07 smithi110 podman[54835]: 2024-09-13 05:38:07.971466455 +0000 UTC m=+1.591790157 container cleanup 4c65de07497b5478247300a584597a847f193db44a54145757a3ffd1772a6b32 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-type=git, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, version=8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream) 2024-09-13T05:38:08.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:07 smithi110 bash[54835]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5 2024-09-13T05:38:08.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:08 smithi110 podman[54857]: 2024-09-13 05:38:08.18989874 +0000 UTC m=+0.852862912 container remove 4c65de07497b5478247300a584597a847f193db44a54145757a3ffd1772a6b32 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, version=8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, RELEASE=HEAD, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.openshift.expose-services=, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8) 2024-09-13T05:38:08.431 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_WARN 1 osds down 2024-09-13T05:38:08.432 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T05:38:08.432 INFO:teuthology.orchestra.run.smithi053.stdout: osd.5 (root=default,host=smithi110) is down 2024-09-13T05:38:08.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 230645 8.55 MB/sec execute 542 sec latency 1015.000 ms 2024-09-13T05:38:08.795 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:08 smithi053 ceph-mon[64292]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:08.796 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:08 smithi053 ceph-mon[64292]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T05:38:08.796 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:08.796 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:08 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/4289835616' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:38:08.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:08 smithi110 ceph-mon[49387]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:08.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:08 smithi110 ceph-mon[49387]: osdmap e130: 8 total, 7 up, 8 in 2024-09-13T05:38:08.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:08.986 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:08 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/4289835616' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:38:08.986 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:08 smithi110 podman[54946]: 2024-09-13 05:38:08.585177501 +0000 UTC m=+0.017752515 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:38:08.986 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:08 smithi110 podman[54946]: 2024-09-13 05:38:08.716196027 +0000 UTC m=+0.148771031 container create 936926783dc0924cd6cbc28eb001a3c878d24ac20cc024486d6a3a4d91e56d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T05:38:09.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:08 smithi110 podman[54946]: 2024-09-13 05:38:08.986003302 +0000 UTC m=+0.418578306 container init 936926783dc0924cd6cbc28eb001a3c878d24ac20cc024486d6a3a4d91e56d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T05:38:09.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:08 smithi110 podman[54946]: 2024-09-13 05:38:08.990360361 +0000 UTC m=+0.422935362 container start 936926783dc0924cd6cbc28eb001a3c878d24ac20cc024486d6a3a4d91e56d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:38:09.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:09 smithi110 podman[54946]: 2024-09-13 05:38:09.085859662 +0000 UTC m=+0.518434658 container attach 936926783dc0924cd6cbc28eb001a3c878d24ac20cc024486d6a3a4d91e56d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T05:38:09.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:09 smithi110 podman[54946]: 2024-09-13 05:38:09.204065071 +0000 UTC m=+0.636640088 container died 936926783dc0924cd6cbc28eb001a3c878d24ac20cc024486d6a3a4d91e56d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-13T05:38:09.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 230847 8.55 MB/sec execute 543 sec latency 59.797 ms 2024-09-13T05:38:09.807 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:09 smithi053 ceph-mon[64292]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:09.807 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:09 smithi053 ceph-mon[64292]: pgmap v224: 97 pgs: 10 peering, 6 stale+active+clean, 81 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-09-13T05:38:09.807 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:09 smithi053 ceph-mon[64292]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T05:38:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:09 smithi110 ceph-mon[49387]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:09.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:09 smithi110 ceph-mon[49387]: pgmap v224: 97 pgs: 10 peering, 6 stale+active+clean, 81 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-09-13T05:38:09.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:09 smithi110 ceph-mon[49387]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T05:38:09.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:09 smithi110 podman[54967]: 2024-09-13 05:38:09.80755763 +0000 UTC m=+0.594150244 container remove 936926783dc0924cd6cbc28eb001a3c878d24ac20cc024486d6a3a4d91e56d4b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-09-13T05:38:09.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:09 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-13T05:38:09.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:09 smithi110 systemd[1]: Stopped Ceph osd.5 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:38:09.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:09 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 32.071s CPU time. 2024-09-13T05:38:10.385 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 systemd[1]: Starting Ceph osd.5 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:38:10.385 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 podman[55070]: 2024-09-13 05:38:10.162870045 +0000 UTC m=+0.019014760 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:38:10.385 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 podman[55070]: 2024-09-13 05:38:10.275683426 +0000 UTC m=+0.131828137 container create 170d155d781b8c36ca5f983736c19db0884544f7d459c61bb65ed0a906bcf5c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:38:10.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 231187 8.55 MB/sec execute 544 sec latency 43.710 ms 2024-09-13T05:38:10.815 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:10 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 73/1641 objects degraded (4.449%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:10 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 73/1641 objects degraded (4.449%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:10.994 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 podman[55070]: 2024-09-13 05:38:10.67858986 +0000 UTC m=+0.534734569 container init 170d155d781b8c36ca5f983736c19db0884544f7d459c61bb65ed0a906bcf5c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T05:38:10.994 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 podman[55070]: 2024-09-13 05:38:10.682571048 +0000 UTC m=+0.538715750 container start 170d155d781b8c36ca5f983736c19db0884544f7d459c61bb65ed0a906bcf5c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-09-13T05:38:10.994 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 podman[55070]: 2024-09-13 05:38:10.770324477 +0000 UTC m=+0.626469181 container attach 170d155d781b8c36ca5f983736c19db0884544f7d459c61bb65ed0a906bcf5c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T05:38:10.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:10.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 bash[55070]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:10.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:10.995 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:10 smithi110 bash[55070]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:11.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 231416 8.54 MB/sec execute 545 sec latency 72.149 ms 2024-09-13T05:38:11.828 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:11 smithi053 ceph-mon[64292]: pgmap v225: 97 pgs: 4 active+undersized, 10 peering, 1 stale+active+clean, 10 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.8 MiB/s wr, 376 op/s; 73/1641 objects degraded (4.449%) 2024-09-13T05:38:11.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:11 smithi110 ceph-mon[49387]: pgmap v225: 97 pgs: 4 active+undersized, 10 peering, 1 stale+active+clean, 10 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.8 MiB/s wr, 376 op/s; 73/1641 objects degraded (4.449%) 2024-09-13T05:38:12.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 231705 8.54 MB/sec execute 546 sec latency 88.578 ms 2024-09-13T05:38:13.466 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T05:38:13.466 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 bash[55070]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T05:38:13.466 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T05:38:13.466 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 bash[55070]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T05:38:13.467 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T05:38:13.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 232005 8.53 MB/sec execute 547 sec latency 64.786 ms 2024-09-13T05:38:13.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:13 smithi053 ceph-mon[64292]: pgmap v226: 97 pgs: 7 active+undersized, 4 peering, 20 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.3 MiB/s wr, 253 op/s; 165/1680 objects degraded (9.821%) 2024-09-13T05:38:13.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:13.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:13.614 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:13 smithi053 ceph-mon[64292]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T05:38:13.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-mon[49387]: pgmap v226: 97 pgs: 7 active+undersized, 4 peering, 20 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.3 MiB/s wr, 253 op/s; 165/1680 objects degraded (9.821%) 2024-09-13T05:38:13.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:13.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:13.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-mon[49387]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T05:38:13.744 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 bash[55070]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 bash[55070]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 bash[55070]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 bash[55070]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate[55085]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 bash[55070]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T05:38:13.745 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:13 smithi110 podman[55070]: 2024-09-13 05:38:13.503106589 +0000 UTC m=+3.359251309 container died 170d155d781b8c36ca5f983736c19db0884544f7d459c61bb65ed0a906bcf5c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T05:38:14.454 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 podman[55241]: 2024-09-13 05:38:14.100493666 +0000 UTC m=+0.591478988 container remove 170d155d781b8c36ca5f983736c19db0884544f7d459c61bb65ed0a906bcf5c9 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T05:38:14.454 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 podman[55314]: 2024-09-13 05:38:14.219376588 +0000 UTC m=+0.017238448 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:38:14.455 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 podman[55314]: 2024-09-13 05:38:14.334898414 +0000 UTC m=+0.132760271 container create caa8bc291506e2a6499efc73cca5fc6d86db7ebd44570ff5571af8305b8dea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T05:38:14.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 232405 8.53 MB/sec execute 548 sec latency 36.935 ms 2024-09-13T05:38:14.744 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 podman[55314]: 2024-09-13 05:38:14.595937748 +0000 UTC m=+0.393799619 container init caa8bc291506e2a6499efc73cca5fc6d86db7ebd44570ff5571af8305b8dea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T05:38:14.744 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 podman[55314]: 2024-09-13 05:38:14.599378387 +0000 UTC m=+0.397240247 container start caa8bc291506e2a6499efc73cca5fc6d86db7ebd44570ff5571af8305b8dea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-13T05:38:14.744 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 ceph-osd[55333]: -- 172.21.15.110:0/2322283684 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556e73c381e0 con 0x556e73bc8000 2024-09-13T05:38:14.744 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 bash[55314]: caa8bc291506e2a6499efc73cca5fc6d86db7ebd44570ff5571af8305b8dea16 2024-09-13T05:38:14.744 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:14 smithi110 systemd[1]: Started Ceph osd.5 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:38:15.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 232759 8.53 MB/sec execute 549 sec latency 63.753 ms 2024-09-13T05:38:15.922 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:15 smithi053 ceph-mon[64292]: pgmap v227: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 185/1665 objects degraded (11.111%) 2024-09-13T05:38:15.922 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:15.922 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:15.922 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:38:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:15 smithi110 ceph-mon[49387]: pgmap v227: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 185/1665 objects degraded (11.111%) 2024-09-13T05:38:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:16.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:38:16.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 233085 8.53 MB/sec execute 550 sec latency 38.864 ms 2024-09-13T05:38:16.732 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:16 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:38:16.469+0000 7fde4aa23640 -1 Falling back to public interface 2024-09-13T05:38:16.942 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:16 smithi053 ceph-mon[64292]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 443 op/s; 182/1626 objects degraded (11.193%) 2024-09-13T05:38:16.942 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:16 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 182/1626 objects degraded (11.193%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:16 smithi110 ceph-mon[49387]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 356 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 443 op/s; 182/1626 objects degraded (11.193%) 2024-09-13T05:38:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:16 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 182/1626 objects degraded (11.193%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:17.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 233553 8.54 MB/sec execute 551 sec latency 61.295 ms 2024-09-13T05:38:18.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 234001 8.56 MB/sec execute 552 sec latency 52.677 ms 2024-09-13T05:38:19.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 234314 8.56 MB/sec execute 553 sec latency 75.703 ms 2024-09-13T05:38:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:19 smithi053 ceph-mon[64292]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 429 op/s; 177/1611 objects degraded (10.987%) 2024-09-13T05:38:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:19.868 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:19 smithi110 ceph-mon[49387]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 429 op/s; 177/1611 objects degraded (10.987%) 2024-09-13T05:38:19.868 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:19.869 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:19.869 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:19.869 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:20.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 234592 8.56 MB/sec execute 554 sec latency 51.991 ms 2024-09-13T05:38:21.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 234854 8.56 MB/sec execute 555 sec latency 54.627 ms 2024-09-13T05:38:21.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:21 smithi110 ceph-mon[49387]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.2 MiB/s wr, 444 op/s; 184/1629 objects degraded (11.295%) 2024-09-13T05:38:21.794 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:21 smithi053 ceph-mon[64292]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.2 MiB/s wr, 444 op/s; 184/1629 objects degraded (11.295%) 2024-09-13T05:38:22.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 235077 8.55 MB/sec execute 556 sec latency 71.158 ms 2024-09-13T05:38:22.823 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:22 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 184/1629 objects degraded (11.295%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:22.968 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:22 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:38:22.771+0000 7fde4aa23640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-13T05:38:22.968 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:22 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 184/1629 objects degraded (11.295%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:23.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 235279 8.54 MB/sec execute 557 sec latency 73.340 ms 2024-09-13T05:38:23.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:23 smithi053 ceph-mon[64292]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 518 op/s; 189/1713 objects degraded (11.033%) 2024-09-13T05:38:23.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:23 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:23.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:23 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:23.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:23 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:38:23.756 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:23 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:38:23.757 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:23 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:23.757 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:23 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-mon[49387]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 518 op/s; 189/1713 objects degraded (11.033%) 2024-09-13T05:38:23.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:23.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:23.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:38:23.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:38:23.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:23.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:24.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 235599 8.54 MB/sec execute 558 sec latency 61.115 ms 2024-09-13T05:38:24.494 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:23 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:38:23.996+0000 7fde4aa23640 -1 osd.5 128 log_to_monitors true 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='osd.5 [v2:172.21.15.110:6808/2073245546,v1:172.21.15.110:6809/2073245546]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T05:38:24.781 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:24 smithi053 ceph-mon[64292]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='osd.5 [v2:172.21.15.110:6808/2073245546,v1:172.21.15.110:6809/2073245546]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T05:38:24.975 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-mon[49387]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T05:38:25.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:38:24 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:38:24.973+0000 7fde3e7c5640 -1 osd.5 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:38:25.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 235891 8.54 MB/sec execute 559 sec latency 54.610 ms 2024-09-13T05:38:25.824 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:25 smithi053 ceph-mon[64292]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 181/1698 objects degraded (10.660%) 2024-09-13T05:38:25.824 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:25 smithi053 ceph-mon[64292]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T05:38:25.824 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:25 smithi053 ceph-mon[64292]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T05:38:25.824 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:25 smithi053 ceph-mon[64292]: from='osd.5 [v2:172.21.15.110:6808/2073245546,v1:172.21.15.110:6809/2073245546]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:38:25.824 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:25 smithi053 ceph-mon[64292]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:38:25.824 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:25 smithi053 ceph-mon[64292]: from='osd.5 ' entity='osd.5' 2024-09-13T05:38:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:25 smithi110 ceph-mon[49387]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 181/1698 objects degraded (10.660%) 2024-09-13T05:38:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:25 smithi110 ceph-mon[49387]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T05:38:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:25 smithi110 ceph-mon[49387]: osdmap e131: 8 total, 7 up, 8 in 2024-09-13T05:38:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:25 smithi110 ceph-mon[49387]: from='osd.5 [v2:172.21.15.110:6808/2073245546,v1:172.21.15.110:6809/2073245546]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:38:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:25 smithi110 ceph-mon[49387]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:38:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:25 smithi110 ceph-mon[49387]: from='osd.5 ' entity='osd.5' 2024-09-13T05:38:26.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 236227 8.54 MB/sec execute 560 sec latency 387.312 ms 2024-09-13T05:38:26.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:26 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:38:26.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:26 smithi110 ceph-mon[49387]: osd.5 [v2:172.21.15.110:6808/2073245546,v1:172.21.15.110:6809/2073245546] boot 2024-09-13T05:38:26.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:26 smithi110 ceph-mon[49387]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T05:38:26.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:26 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T05:38:26.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:26 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:38:26.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:26 smithi053 ceph-mon[64292]: osd.5 [v2:172.21.15.110:6808/2073245546,v1:172.21.15.110:6809/2073245546] boot 2024-09-13T05:38:26.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:26 smithi053 ceph-mon[64292]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T05:38:26.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:26 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T05:38:27.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 236347 8.53 MB/sec execute 561 sec latency 972.788 ms 2024-09-13T05:38:27.808 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:27 smithi053 ceph-mon[64292]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 174/1656 objects degraded (10.507%) 2024-09-13T05:38:27.927 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:27 smithi053 ceph-mon[64292]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T05:38:27.927 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:27 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 174/1656 objects degraded (10.507%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:27.927 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:27.927 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:27 smithi110 ceph-mon[49387]: pgmap v235: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.4 MiB/s wr, 432 op/s; 174/1656 objects degraded (10.507%) 2024-09-13T05:38:27.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:27 smithi110 ceph-mon[49387]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T05:38:27.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:27 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 174/1656 objects degraded (10.507%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:27.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:27.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:28.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 236507 8.52 MB/sec execute 562 sec latency 867.488 ms 2024-09-13T05:38:29.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:28 smithi053 ceph-mon[64292]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T05:38:29.106 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:29 smithi053 ceph-mon[64292]: pgmap v238: 97 pgs: 6 peering, 5 active+undersized, 19 active+undersized+degraded, 67 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.0 MiB/s wr, 431 op/s; 142/1551 objects degraded (9.155%) 2024-09-13T05:38:29.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 236854 8.52 MB/sec execute 563 sec latency 63.452 ms 2024-09-13T05:38:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:29 smithi110 ceph-mon[49387]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T05:38:29.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:29 smithi110 ceph-mon[49387]: pgmap v238: 97 pgs: 6 peering, 5 active+undersized, 19 active+undersized+degraded, 67 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.0 MiB/s wr, 431 op/s; 142/1551 objects degraded (9.155%) 2024-09-13T05:38:30.124 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:30 smithi053 ceph-mon[64292]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T05:38:30.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 237170 8.52 MB/sec execute 564 sec latency 440.264 ms 2024-09-13T05:38:30.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:30 smithi110 ceph-mon[49387]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T05:38:31.177 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:31 smithi053 ceph-mon[64292]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T05:38:31.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:31 smithi053 ceph-mon[64292]: pgmap v241: 97 pgs: 1 active+undersized+remapped, 6 peering, 1 active+undersized, 3 active+undersized+degraded, 86 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 593 op/s; 33/1527 objects degraded (2.161%); 1.4 MiB/s, 25 objects/s recovering 2024-09-13T05:38:31.178 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:31 smithi053 ceph-mon[64292]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T05:38:31.461 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:31 smithi110 ceph-mon[49387]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T05:38:31.462 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:31 smithi110 ceph-mon[49387]: pgmap v241: 97 pgs: 1 active+undersized+remapped, 6 peering, 1 active+undersized, 3 active+undersized+degraded, 86 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 593 op/s; 33/1527 objects degraded (2.161%); 1.4 MiB/s, 25 objects/s recovering 2024-09-13T05:38:31.462 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:31 smithi110 ceph-mon[49387]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T05:38:31.480 INFO:teuthology.orchestra.run.smithi053.stdout: 5 237394 8.52 MB/sec execute 565 sec latency 946.721 ms 2024-09-13T05:38:32.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 237636 8.52 MB/sec execute 566 sec latency 48.375 ms 2024-09-13T05:38:33.254 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:33 smithi053 ceph-mon[64292]: pgmap v243: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+undersized+degraded, 94 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.9 MiB/s wr, 310 op/s; 14/1578 objects degraded (0.887%); 2.6 MiB/s, 48 objects/s recovering 2024-09-13T05:38:33.254 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:33 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 14/1578 objects degraded (0.887%), 1 pg degraded (PG_DEGRADED) 2024-09-13T05:38:33.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 237933 8.53 MB/sec execute 567 sec latency 52.948 ms 2024-09-13T05:38:33.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:33 smithi110 ceph-mon[49387]: pgmap v243: 97 pgs: 1 active+undersized+remapped, 1 peering, 1 active+undersized+degraded, 94 active+clean; 346 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.9 MiB/s wr, 310 op/s; 14/1578 objects degraded (0.887%); 2.6 MiB/s, 48 objects/s recovering 2024-09-13T05:38:33.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:33 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 14/1578 objects degraded (0.887%), 1 pg degraded (PG_DEGRADED) 2024-09-13T05:38:34.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 238244 8.52 MB/sec execute 568 sec latency 62.727 ms 2024-09-13T05:38:35.281 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:35 smithi053 ceph-mon[64292]: pgmap v244: 97 pgs: 1 peering, 1 active+undersized+degraded, 95 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.6 MiB/s wr, 254 op/s; 14/1632 objects degraded (0.858%); 1.7 MiB/s, 33 objects/s recovering 2024-09-13T05:38:35.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 238554 8.52 MB/sec execute 569 sec latency 50.115 ms 2024-09-13T05:38:35.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:35 smithi110 ceph-mon[49387]: pgmap v244: 97 pgs: 1 peering, 1 active+undersized+degraded, 95 active+clean; 347 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.6 MiB/s wr, 254 op/s; 14/1632 objects degraded (0.858%); 1.7 MiB/s, 33 objects/s recovering 2024-09-13T05:38:36.319 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:36 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/1632 objects degraded (0.858%), 1 pg degraded) 2024-09-13T05:38:36.319 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:36 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:38:36.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 238733 8.51 MB/sec execute 570 sec latency 58.772 ms 2024-09-13T05:38:36.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:36 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/1632 objects degraded (0.858%), 1 pg degraded) 2024-09-13T05:38:36.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:36 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:38:37.343 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:37 smithi053 ceph-mon[64292]: pgmap v245: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 317 op/s; 939 KiB/s, 26 objects/s recovering 2024-09-13T05:38:37.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 238987 8.51 MB/sec execute 571 sec latency 75.465 ms 2024-09-13T05:38:37.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:37 smithi110 ceph-mon[49387]: pgmap v245: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 317 op/s; 939 KiB/s, 26 objects/s recovering 2024-09-13T05:38:38.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 239233 8.50 MB/sec execute 572 sec latency 63.673 ms 2024-09-13T05:38:38.975 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:38:39.372 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:39 smithi110 ceph-mon[49387]: pgmap v246: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 359 op/s; 645 KiB/s, 15 objects/s recovering 2024-09-13T05:38:39.372 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T05:38:39.372 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:39 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T05:38:39.372 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:39 smithi110 ceph-mon[49387]: Upgrade: osd.6 is safe to restart 2024-09-13T05:38:39.374 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:39 smithi053 ceph-mon[64292]: pgmap v246: 97 pgs: 97 active+clean; 350 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 359 op/s; 645 KiB/s, 15 objects/s recovering 2024-09-13T05:38:39.374 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T05:38:39.374 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:39 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T05:38:39.375 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:39 smithi053 ceph-mon[64292]: Upgrade: osd.6 is safe to restart 2024-09-13T05:38:39.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 239523 8.50 MB/sec execute 573 sec latency 47.607 ms 2024-09-13T05:38:39.897 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:38:39.897 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (8m) 99s ago 18m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:38:39.897 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (4m) 99s ago 18m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:38:39.897 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (4m) 21s ago 17m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:38:39.897 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (7m) 99s ago 17m 44.3M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:38:39.897 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (12m) 99s ago 12m 38.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (12m) 21s ago 12m 16.8M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (7m) 99s ago 19m 518M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (6m) 21s ago 17m 456M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (6m) 99s ago 19m 52.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (5m) 21s ago 17m 45.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (11m) 99s ago 11m 279M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (11m) 21s ago 11m 90.8M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (7m) 99s ago 17m 10.2M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (7m) 21s ago 17m 12.1M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (4m) 99s ago 16m 307M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (3m) 99s ago 16m 248M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (2m) 99s ago 15m 195M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (108s) 99s ago 15m 14.2M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (62s) 21s ago 14m 258M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:38:39.898 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (25s) 21s ago 14m 14.4M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 caa8bc291506 2024-09-13T05:38:39.899 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (14m) 21s ago 14m 471M 2704M 17.2.0 e1d6a67b021e 78f371548dfd 2024-09-13T05:38:39.899 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (13m) 21s ago 13m 751M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:38:39.899 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (6m) 99s ago 17m 54.0M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:38:40.396 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:40 smithi053 ceph-mon[64292]: from='client.44277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:40.396 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:40 smithi053 ceph-mon[64292]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:40.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 239804 8.50 MB/sec execute 574 sec latency 64.289 ms 2024-09-13T05:38:40.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:40 smithi110 ceph-mon[49387]: from='client.44277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:40.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:40 smithi110 ceph-mon[49387]: from='client.34392 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:40.575 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:38:40.575 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:38:40.575 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 6 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:38:40.576 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:38:40.577 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T05:38:40.577 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 10 2024-09-13T05:38:40.577 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:38:40.577 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:38:41.146 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "12/23 daemons upgraded", 2024-09-13T05:38:41.147 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:38:41.147 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:38:41.147 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:38:41.435 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:41.436 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: pgmap v247: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 374 op/s; 520 KiB/s, 12 objects/s recovering 2024-09-13T05:38:41.436 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: Upgrade: Updating osd.6 2024-09-13T05:38:41.436 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:41.436 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T05:38:41.436 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:38:41.436 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: Deploying daemon osd.6 on smithi110 2024-09-13T05:38:41.436 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:41 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3210526439' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:41.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 240142 8.50 MB/sec execute 575 sec latency 46.772 ms 2024-09-13T05:38:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: from='client.34396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: pgmap v247: 97 pgs: 97 active+clean; 348 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 374 op/s; 520 KiB/s, 12 objects/s recovering 2024-09-13T05:38:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: Upgrade: Updating osd.6 2024-09-13T05:38:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T05:38:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:38:41.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: Deploying daemon osd.6 on smithi110 2024-09-13T05:38:41.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:41 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3210526439' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:38:41.831 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:38:42.453 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:42 smithi053 ceph-mon[64292]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:42.453 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:42 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/2698234309' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:38:42.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 240421 8.49 MB/sec execute 576 sec latency 66.164 ms 2024-09-13T05:38:42.669 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:42 smithi110 ceph-mon[49387]: from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:38:42.669 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:42 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/2698234309' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:38:43.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 240777 8.50 MB/sec execute 577 sec latency 50.788 ms 2024-09-13T05:38:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:43 smithi110 ceph-mon[49387]: pgmap v248: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 360 op/s; 467 KiB/s, 6 objects/s recovering 2024-09-13T05:38:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:43.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:43.745 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:43.762 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:43 smithi053 ceph-mon[64292]: pgmap v248: 97 pgs: 97 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.0 MiB/s wr, 360 op/s; 467 KiB/s, 6 objects/s recovering 2024-09-13T05:38:43.762 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:43.762 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:43.762 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:44.053 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:43 smithi110 systemd[1]: Stopping Ceph osd.6 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:38:44.485 INFO:teuthology.orchestra.run.smithi053.stdout: 5 241005 8.49 MB/sec execute 578 sec latency 61.280 ms 2024-09-13T05:38:44.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[35939]: 2024-09-13T05:38:44.098+0000 7fe4a40c8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:38:44.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[35939]: 2024-09-13T05:38:44.098+0000 7fe4a40c8700 -1 osd.6 137 *** Got signal Terminated *** 2024-09-13T05:38:44.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[35939]: 2024-09-13T05:38:44.098+0000 7fe4a40c8700 -1 osd.6 137 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:38:44.799 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:44 smithi053 ceph-mon[64292]: osd.6 marked itself down and dead 2024-09-13T05:38:44.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:44 smithi110 ceph-mon[49387]: osd.6 marked itself down and dead 2024-09-13T05:38:45.482 INFO:teuthology.orchestra.run.smithi053.stdout: 5 241116 8.49 MB/sec execute 579 sec latency 783.579 ms 2024-09-13T05:38:45.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:45 smithi110 podman[56890]: 2024-09-13 05:38:45.225826253 +0000 UTC m=+1.253591922 container died 78f371548dfda799c0ba2e48a84cd582a28125f4f256dcaa14bf706acf8f4844 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.component=centos-stream-container, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., ceph=True, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T05:38:45.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:45 smithi053 ceph-mon[64292]: pgmap v249: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 2.0 KiB/s, 1 objects/s recovering 2024-09-13T05:38:45.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:45 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:38:45.826 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:45 smithi053 ceph-mon[64292]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T05:38:45.966 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:45 smithi110 ceph-mon[49387]: pgmap v249: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.0 MiB/s wr, 372 op/s; 2.0 KiB/s, 1 objects/s recovering 2024-09-13T05:38:45.966 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:45 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:38:45.966 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:45 smithi110 ceph-mon[49387]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T05:38:45.967 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:45 smithi110 podman[56890]: 2024-09-13 05:38:45.729844203 +0000 UTC m=+1.757609873 container cleanup 78f371548dfda799c0ba2e48a84cd582a28125f4f256dcaa14bf706acf8f4844 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, architecture=x86_64, vcs-type=git, version=8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, release=754, ceph=True, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870) 2024-09-13T05:38:45.967 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:45 smithi110 bash[56890]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6 2024-09-13T05:38:46.244 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:45 smithi110 podman[56916]: 2024-09-13 05:38:45.966322996 +0000 UTC m=+0.737787238 container remove 78f371548dfda799c0ba2e48a84cd582a28125f4f256dcaa14bf706acf8f4844 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, architecture=x86_64, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, release=754, GIT_CLEAN=True, com.redhat.component=centos-stream-container, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-type=git, version=8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T05:38:46.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 241374 8.49 MB/sec execute 580 sec latency 911.203 ms 2024-09-13T05:38:46.574 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:46 smithi110 podman[57005]: 2024-09-13 05:38:46.223370644 +0000 UTC m=+0.016854914 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:38:46.574 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:46 smithi110 podman[57005]: 2024-09-13 05:38:46.335062568 +0000 UTC m=+0.128546811 container create 447f8ea66e8b9a1e316aaafb137902b16285eb4b376f5dd92f21b4519942d07c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T05:38:46.574 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:46 smithi110 podman[57005]: 2024-09-13 05:38:46.573970188 +0000 UTC m=+0.367454433 container init 447f8ea66e8b9a1e316aaafb137902b16285eb4b376f5dd92f21b4519942d07c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-13T05:38:46.836 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:46 smithi053 ceph-mon[64292]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T05:38:46.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:46 smithi110 ceph-mon[49387]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T05:38:46.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:46 smithi110 podman[57005]: 2024-09-13 05:38:46.577454171 +0000 UTC m=+0.370938415 container start 447f8ea66e8b9a1e316aaafb137902b16285eb4b376f5dd92f21b4519942d07c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T05:38:46.995 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:46 smithi110 podman[57005]: 2024-09-13 05:38:46.665813159 +0000 UTC m=+0.459297404 container attach 447f8ea66e8b9a1e316aaafb137902b16285eb4b376f5dd92f21b4519942d07c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T05:38:46.995 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:46 smithi110 podman[57005]: 2024-09-13 05:38:46.797635559 +0000 UTC m=+0.591119813 container died 447f8ea66e8b9a1e316aaafb137902b16285eb4b376f5dd92f21b4519942d07c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:38:47.481 INFO:teuthology.orchestra.run.smithi053.stdout: 5 241667 8.49 MB/sec execute 581 sec latency 63.898 ms 2024-09-13T05:38:47.666 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:47 smithi110 ceph-mon[49387]: pgmap v252: 97 pgs: 7 peering, 7 stale+active+clean, 1 active+undersized+degraded, 82 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 433 op/s; 10/1578 objects degraded (0.634%) 2024-09-13T05:38:47.666 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:47 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 10/1578 objects degraded (0.634%), 1 pg degraded (PG_DEGRADED) 2024-09-13T05:38:47.846 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:47 smithi053 ceph-mon[64292]: pgmap v252: 97 pgs: 7 peering, 7 stale+active+clean, 1 active+undersized+degraded, 82 active+clean; 349 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 433 op/s; 10/1578 objects degraded (0.634%) 2024-09-13T05:38:47.846 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:47 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 10/1578 objects degraded (0.634%), 1 pg degraded (PG_DEGRADED) 2024-09-13T05:38:47.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:47 smithi110 podman[57028]: 2024-09-13 05:38:47.666458624 +0000 UTC m=+0.862377527 container remove 447f8ea66e8b9a1e316aaafb137902b16285eb4b376f5dd92f21b4519942d07c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T05:38:47.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:47 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-13T05:38:47.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:47 smithi110 systemd[1]: Stopped Ceph osd.6 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:38:47.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:47 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.6.service: Consumed 4min 208ms CPU time. 2024-09-13T05:38:47.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:47 smithi110 systemd[1]: Starting Ceph osd.6 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:38:48.485 INFO:teuthology.orchestra.run.smithi053.stdout: 5 241979 8.49 MB/sec execute 582 sec latency 72.915 ms 2024-09-13T05:38:48.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 podman[57126]: 2024-09-13 05:38:48.117844655 +0000 UTC m=+0.112631884 container create 4382c3c2dfd71fe00ff68f914e236b9f8304cceac0e7bf088441a8185362f1da (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:38:48.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 podman[57126]: 2024-09-13 05:38:48.024087187 +0000 UTC m=+0.018874427 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:38:48.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 podman[57126]: 2024-09-13 05:38:48.68761455 +0000 UTC m=+0.682401793 container init 4382c3c2dfd71fe00ff68f914e236b9f8304cceac0e7bf088441a8185362f1da (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-13T05:38:48.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 podman[57126]: 2024-09-13 05:38:48.691543263 +0000 UTC m=+0.686330495 container start 4382c3c2dfd71fe00ff68f914e236b9f8304cceac0e7bf088441a8185362f1da (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T05:38:48.994 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 podman[57126]: 2024-09-13 05:38:48.795989534 +0000 UTC m=+0.790776782 container attach 4382c3c2dfd71fe00ff68f914e236b9f8304cceac0e7bf088441a8185362f1da (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:38:48.995 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:48.995 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 bash[57126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:48.995 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:48.995 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:48 smithi110 bash[57126]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:38:49.495 INFO:teuthology.orchestra.run.smithi053.stdout: 5 242245 8.49 MB/sec execute 583 sec latency 178.231 ms 2024-09-13T05:38:49.679 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:49 smithi053 ceph-mon[64292]: pgmap v253: 97 pgs: 8 active+undersized, 7 peering, 1 stale+active+clean, 15 active+undersized+degraded, 66 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.0 MiB/s wr, 366 op/s; 108/1593 objects degraded (6.780%) 2024-09-13T05:38:49.895 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:49 smithi110 ceph-mon[49387]: pgmap v253: 97 pgs: 8 active+undersized, 7 peering, 1 stale+active+clean, 15 active+undersized+degraded, 66 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.0 MiB/s wr, 366 op/s; 108/1593 objects degraded (6.780%) 2024-09-13T05:38:50.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 242516 8.48 MB/sec execute 584 sec latency 54.071 ms 2024-09-13T05:38:51.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T05:38:51.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 bash[57126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T05:38:51.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T05:38:51.495 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 bash[57126]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T05:38:51.495 INFO:teuthology.orchestra.run.smithi053.stdout: 5 242868 8.48 MB/sec execute 585 sec latency 46.647 ms 2024-09-13T05:38:51.705 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:51 smithi053 ceph-mon[64292]: pgmap v254: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 464 op/s; 184/1605 objects degraded (11.464%) 2024-09-13T05:38:51.943 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-mon[49387]: pgmap v254: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 464 op/s; 184/1605 objects degraded (11.464%) 2024-09-13T05:38:51.943 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T05:38:51.943 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 bash[57126]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T05:38:51.943 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T05:38:51.943 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 bash[57126]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T05:38:51.944 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T05:38:51.944 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 bash[57126]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T05:38:51.944 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T05:38:51.944 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 bash[57126]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T05:38:51.944 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate[57140]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T05:38:51.944 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 bash[57126]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T05:38:51.944 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:51 smithi110 podman[57126]: 2024-09-13 05:38:51.559329376 +0000 UTC m=+3.554116615 container died 4382c3c2dfd71fe00ff68f914e236b9f8304cceac0e7bf088441a8185362f1da (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:38:52.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 243169 8.47 MB/sec execute 586 sec latency 52.675 ms 2024-09-13T05:38:52.647 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 podman[57295]: 2024-09-13 05:38:52.344850336 +0000 UTC m=+0.774961638 container remove 4382c3c2dfd71fe00ff68f914e236b9f8304cceac0e7bf088441a8185362f1da (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:38:52.647 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 podman[57353]: 2024-09-13 05:38:52.486725434 +0000 UTC m=+0.023299453 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:38:52.647 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 podman[57353]: 2024-09-13 05:38:52.587662186 +0000 UTC m=+0.124236191 container create 260b18297eb59de6e30c12a948bb8bd6cdb0ac11bf219da257eea037f29c1b40 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:38:52.715 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:52 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 184/1605 objects degraded (11.464%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:52.934 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:52 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 184/1605 objects degraded (11.464%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:52.935 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 podman[57353]: 2024-09-13 05:38:52.83464398 +0000 UTC m=+0.371217982 container init 260b18297eb59de6e30c12a948bb8bd6cdb0ac11bf219da257eea037f29c1b40 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:38:52.935 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 podman[57353]: 2024-09-13 05:38:52.839368895 +0000 UTC m=+0.375942893 container start 260b18297eb59de6e30c12a948bb8bd6cdb0ac11bf219da257eea037f29c1b40 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=) 2024-09-13T05:38:52.935 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 ceph-osd[57370]: -- 172.21.15.110:0/1882349263 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563cf638e1e0 con 0x563cf638a000 2024-09-13T05:38:53.244 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 bash[57353]: 260b18297eb59de6e30c12a948bb8bd6cdb0ac11bf219da257eea037f29c1b40 2024-09-13T05:38:53.244 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:52 smithi110 systemd[1]: Started Ceph osd.6 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:38:53.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 243454 8.48 MB/sec execute 587 sec latency 69.639 ms 2024-09-13T05:38:53.724 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:53 smithi053 ceph-mon[64292]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 186/1650 objects degraded (11.273%) 2024-09-13T05:38:53.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:53.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:53.725 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:53 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:38:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:53 smithi110 ceph-mon[49387]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 4.2 MiB/s wr, 407 op/s; 186/1650 objects degraded (11.273%) 2024-09-13T05:38:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:53.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:53 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:38:54.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 243710 8.48 MB/sec execute 588 sec latency 58.306 ms 2024-09-13T05:38:54.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:38:54 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:38:54.458+0000 7f7a87604640 -1 Falling back to public interface 2024-09-13T05:38:55.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 244007 8.48 MB/sec execute 589 sec latency 75.071 ms 2024-09-13T05:38:55.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:55 smithi110 ceph-mon[49387]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 178/1611 objects degraded (11.049%) 2024-09-13T05:38:55.775 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:55 smithi053 ceph-mon[64292]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.0 MiB/s wr, 379 op/s; 178/1611 objects degraded (11.049%) 2024-09-13T05:38:56.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 244378 8.48 MB/sec execute 590 sec latency 51.167 ms 2024-09-13T05:38:57.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 244638 8.47 MB/sec execute 591 sec latency 44.297 ms 2024-09-13T05:38:57.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:57 smithi053 ceph-mon[64292]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 369 op/s; 177/1620 objects degraded (10.926%) 2024-09-13T05:38:57.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:57.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:57.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:57 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 177/1620 objects degraded (10.926%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:57.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:57.798 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:57 smithi110 ceph-mon[49387]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 369 op/s; 177/1620 objects degraded (10.926%) 2024-09-13T05:38:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:57 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 177/1620 objects degraded (10.926%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:38:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:38:58.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 244891 8.47 MB/sec execute 592 sec latency 56.500 ms 2024-09-13T05:38:58.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:58.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:58.816 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:38:58 smithi053 ceph-mon[64292]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 353 op/s; 187/1629 objects degraded (11.479%) 2024-09-13T05:38:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:38:59.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:38:58 smithi110 ceph-mon[49387]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 353 op/s; 187/1629 objects degraded (11.479%) 2024-09-13T05:38:59.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 245219 8.48 MB/sec execute 593 sec latency 82.198 ms 2024-09-13T05:39:00.244 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:39:00 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:39:00.078+0000 7f7a87604640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-13T05:39:00.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 245539 8.47 MB/sec execute 594 sec latency 60.125 ms 2024-09-13T05:39:01.375 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:01 smithi053 ceph-mon[64292]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.1 MiB/s wr, 421 op/s; 189/1638 objects degraded (11.538%) 2024-09-13T05:39:01.481 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:39:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:39:01.142+0000 7f7a87604640 -1 osd.6 137 log_to_monitors true 2024-09-13T05:39:01.481 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:01 smithi110 ceph-mon[49387]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 347 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.1 MiB/s wr, 421 op/s; 189/1638 objects degraded (11.538%) 2024-09-13T05:39:01.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 245817 8.48 MB/sec execute 595 sec latency 48.907 ms 2024-09-13T05:39:01.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:39:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:39:01.479+0000 7f7a7b3a6640 -1 osd.6 137 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:39:02.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='osd.6 [v2:172.21.15.110:6816/3906415359,v1:172.21.15.110:6817/3906415359]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T05:39:02.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T05:39:02.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T05:39:02.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T05:39:02.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='osd.6 [v2:172.21.15.110:6816/3906415359,v1:172.21.15.110:6817/3906415359]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:02.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:02.390 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='osd.6 ' entity='osd.6' 2024-09-13T05:39:02.391 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.391 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.391 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:02.391 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:39:02.391 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.391 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='osd.6 [v2:172.21.15.110:6816/3906415359,v1:172.21.15.110:6817/3906415359]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T05:39:02.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T05:39:02.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T05:39:02.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T05:39:02.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='osd.6 [v2:172.21.15.110:6816/3906415359,v1:172.21.15.110:6817/3906415359]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='osd.6 ' entity='osd.6' 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:02.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 246008 8.47 MB/sec execute 596 sec latency 88.816 ms 2024-09-13T05:39:03.416 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 473 op/s; 195/1641 objects degraded (11.883%) 2024-09-13T05:39:03.416 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:03.416 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:03.416 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: osd.6 [v2:172.21.15.110:6816/3906415359,v1:172.21.15.110:6817/3906415359] boot 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 189/1638 objects degraded (11.538%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:03.417 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:03 smithi053 ceph-mon[64292]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T05:39:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: pgmap v261: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 473 op/s; 195/1641 objects degraded (11.883%) 2024-09-13T05:39:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: osd.6 [v2:172.21.15.110:6816/3906415359,v1:172.21.15.110:6817/3906415359] boot 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 189/1638 objects degraded (11.538%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:03.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:03 smithi110 ceph-mon[49387]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T05:39:03.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 246196 8.46 MB/sec execute 597 sec latency 829.210 ms 2024-09-13T05:39:04.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 246476 8.46 MB/sec execute 598 sec latency 927.362 ms 2024-09-13T05:39:04.557 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:04 smithi053 ceph-mon[64292]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T05:39:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:04 smithi110 ceph-mon[49387]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T05:39:05.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 246713 8.45 MB/sec execute 599 sec latency 74.452 ms 2024-09-13T05:39:05.721 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:05 smithi053 ceph-mon[64292]: pgmap v265: 97 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 10 active+undersized, 21 active+undersized+degraded, 64 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 4.5 MiB/s wr, 361 op/s; 181/1626 objects degraded (11.132%); 0 B/s, 2 objects/s recovering 2024-09-13T05:39:05.722 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:05 smithi053 ceph-mon[64292]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T05:39:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:05 smithi110 ceph-mon[49387]: pgmap v265: 97 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 10 active+undersized, 21 active+undersized+degraded, 64 active+clean; 350 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 4.5 MiB/s wr, 361 op/s; 181/1626 objects degraded (11.132%); 0 B/s, 2 objects/s recovering 2024-09-13T05:39:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:05 smithi110 ceph-mon[49387]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T05:39:06.496 INFO:teuthology.orchestra.run.smithi053.stdout: 5 cleanup 600 sec 2024-09-13T05:39:06.755 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:06 smithi053 ceph-mon[64292]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T05:39:06.843 INFO:teuthology.orchestra.run.smithi053.stdout: 0 cleanup 600 sec 2024-09-13T05:39:06.843 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:39:06.843 INFO:teuthology.orchestra.run.smithi053.stdout: Operation Count AvgLat MaxLat 2024-09-13T05:39:06.843 INFO:teuthology.orchestra.run.smithi053.stdout: ---------------------------------------- 2024-09-13T05:39:06.843 INFO:teuthology.orchestra.run.smithi053.stdout: NTCreateX 162282 5.149 1041.807 2024-09-13T05:39:06.843 INFO:teuthology.orchestra.run.smithi053.stdout: Close 119094 0.141 10.862 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Rename 6894 10.519 101.103 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Unlink 32850 7.573 1787.253 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Qpathinfo 147079 1.365 1040.335 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Qfileinfo 25670 0.007 0.880 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Qfsinfo 27047 1.047 1338.149 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Sfileinfo 13238 14.574 166.060 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Find 56882 1.025 27.234 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: WriteX 80235 16.707 2599.850 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: ReadX 255302 0.010 7.136 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: LockX 534 0.384 2.772 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: UnlockX 534 0.349 2.872 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: Flush 11388 0.003 0.079 2024-09-13T05:39:06.844 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:39:06.845 INFO:teuthology.orchestra.run.smithi053.stdout:Throughput 8.44909 MB/sec 5 clients 5 procs max_latency=2599.857 ms 2024-09-13T05:39:06.846 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-13T05:39:06.886 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-13T05:39:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:06 smithi110 ceph-mon[49387]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T05:39:07.045 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-13T05:39:07.073 INFO:teuthology.orchestra.run.smithi053.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T05:39:07.073 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:39:07.073 INFO:teuthology.orchestra.run.smithi053.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-13T05:39:07.073 INFO:teuthology.orchestra.run.smithi053.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-13T05:39:07.173 INFO:teuthology.orchestra.run.smithi053.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T05:39:07.173 INFO:teuthology.orchestra.run.smithi053.stdout:releasing clients 2024-09-13T05:39:07.864 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:07 smithi053 ceph-mon[64292]: pgmap v268: 97 pgs: 1 active+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 9 active+undersized, 17 active+undersized+degraded, 68 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s wr, 436 op/s; 151/1566 objects degraded (9.642%); 48 KiB/s, 13 objects/s recovering 2024-09-13T05:39:07.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:07 smithi053 ceph-mon[64292]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T05:39:07.865 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:07 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 151/1566 objects degraded (9.642%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:07 smithi110 ceph-mon[49387]: pgmap v268: 97 pgs: 1 active+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 9 active+undersized, 17 active+undersized+degraded, 68 active+clean; 349 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s wr, 436 op/s; 151/1566 objects degraded (9.642%); 48 KiB/s, 13 objects/s recovering 2024-09-13T05:39:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:07 smithi110 ceph-mon[49387]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T05:39:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:07 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 151/1566 objects degraded (9.642%), 19 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:09.173 INFO:teuthology.orchestra.run.smithi053.stdout: 5 276 21.07 MB/sec execute 1 sec latency 368.478 ms 2024-09-13T05:39:09.681 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:09 smithi053 ceph-mon[64292]: pgmap v270: 97 pgs: 1 active+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 94 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.9 MiB/s wr, 370 op/s; 22/1512 objects degraded (1.455%); 2.5 MiB/s, 54 keys/s, 34 objects/s recovering 2024-09-13T05:39:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:09 smithi110 ceph-mon[49387]: pgmap v270: 97 pgs: 1 active+undersized+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 94 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.9 MiB/s wr, 370 op/s; 22/1512 objects degraded (1.455%); 2.5 MiB/s, 54 keys/s, 34 objects/s recovering 2024-09-13T05:39:10.173 INFO:teuthology.orchestra.run.smithi053.stdout: 5 411 21.07 MB/sec execute 2 sec latency 90.784 ms 2024-09-13T05:39:10.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:10 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1512 objects degraded (1.455%), 2 pgs degraded) 2024-09-13T05:39:10.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:10 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:39:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:10 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1512 objects degraded (1.455%), 2 pgs degraded) 2024-09-13T05:39:10.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:10 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:39:11.173 INFO:teuthology.orchestra.run.smithi053.stdout: 5 520 19.73 MB/sec execute 3 sec latency 142.082 ms 2024-09-13T05:39:11.920 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:11 smithi053 ceph-mon[64292]: pgmap v271: 97 pgs: 97 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 431 op/s; 1.9 MiB/s, 41 keys/s, 29 objects/s recovering 2024-09-13T05:39:11.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:11 smithi110 ceph-mon[49387]: pgmap v271: 97 pgs: 97 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.7 MiB/s wr, 431 op/s; 1.9 MiB/s, 41 keys/s, 29 objects/s recovering 2024-09-13T05:39:12.173 INFO:teuthology.orchestra.run.smithi053.stdout: 5 707 18.89 MB/sec execute 4 sec latency 211.506 ms 2024-09-13T05:39:12.275 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:39:13.152 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (8m) 2m ago 18m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (5m) 2m ago 18m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (5m) 16s ago 18m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (8m) 2m ago 18m 44.3M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (12m) 2m ago 12m 38.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (12m) 16s ago 12m 16.9M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (7m) 2m ago 19m 518M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (6m) 16s ago 17m 456M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (6m) 2m ago 20m 52.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (6m) 16s ago 17m 46.0M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:39:13.153 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (12m) 2m ago 12m 279M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (12m) 16s ago 12m 90.8M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (8m) 2m ago 18m 10.2M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (7m) 16s ago 17m 12.5M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (4m) 2m ago 17m 307M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (3m) 2m ago 16m 248M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (3m) 2m ago 16m 195M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (2m) 2m ago 15m 14.2M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (96s) 16s ago 15m 296M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (58s) 16s ago 14m 254M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 caa8bc291506 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (20s) 16s ago 14m 12.1M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 260b18297eb5 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (14m) 16s ago 14m 549M 2704M 17.2.0 e1d6a67b021e 629528ca2e76 2024-09-13T05:39:13.154 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (7m) 2m ago 17m 54.0M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:39:13.173 INFO:teuthology.orchestra.run.smithi053.stdout: 5 cleanup 5 sec 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: 0 cleanup 5 sec 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: Operation Count AvgLat MaxLat 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: ---------------------------------------- 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: NTCreateX 534 11.362 124.022 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: Close 498 16.498 211.499 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: Rename 15 37.288 63.999 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: Unlink 37 3.740 17.491 2024-09-13T05:39:13.453 INFO:teuthology.orchestra.run.smithi053.stdout: Qpathinfo 108 1.564 46.554 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: Qfileinfo 338 18.873 158.335 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: Qfsinfo 78 0.398 1.841 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: Sfileinfo 68 18.808 70.610 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: Find 94 0.787 1.834 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: WriteX 1641 0.027 0.141 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: ReadX 344 0.020 0.717 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: Flush 103 20.469 71.016 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:39:13.454 INFO:teuthology.orchestra.run.smithi053.stdout:Throughput 18.8871 MB/sec 5 clients 5 procs max_latency=368.478 ms 2024-09-13T05:39:13.455 DEBUG:teuthology.parallel:result is None 2024-09-13T05:39:13.689 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:39:13.689 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:39:13.689 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 7 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:39:13.690 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T05:39:13.691 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 11 2024-09-13T05:39:13.691 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:39:13.691 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:39:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:13 smithi110 ceph-mon[49387]: pgmap v272: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 509 op/s; 1.5 MiB/s, 33 keys/s, 23 objects/s recovering 2024-09-13T05:39:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:13 smithi110 ceph-mon[49387]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:13.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:13 smithi110 ceph-mon[49387]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:13 smithi053 ceph-mon[64292]: pgmap v272: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 509 op/s; 1.5 MiB/s, 33 keys/s, 23 objects/s recovering 2024-09-13T05:39:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:13 smithi053 ceph-mon[64292]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:13 smithi053 ceph-mon[64292]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:14.052 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:39:14.052 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:39:14.052 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:39:14.053 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:39:14.053 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:39:14.053 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:39:14.053 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:39:14.053 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:39:14.054 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:39:14.054 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "13/23 daemons upgraded", 2024-09-13T05:39:14.054 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T05:39:14.054 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:39:14.054 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:39:14.495 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:39:14.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:14 smithi053 ceph-mon[64292]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:14.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:14 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3073888605' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:14.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:14 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/2776079558' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:39:14.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:14 smithi110 ceph-mon[49387]: from='client.34422 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:14.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:14 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3073888605' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:14.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:14 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/2776079558' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:39:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:15 smithi110 ceph-mon[49387]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:15 smithi110 ceph-mon[49387]: pgmap v273: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 13 MiB/s wr, 347 op/s; 1.3 MiB/s, 29 keys/s, 16 objects/s recovering 2024-09-13T05:39:15.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:15 smithi053 ceph-mon[64292]: from='client.34430 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:15.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:15 smithi053 ceph-mon[64292]: pgmap v273: 97 pgs: 97 active+clean; 330 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 13 MiB/s wr, 347 op/s; 1.3 MiB/s, 29 keys/s, 16 objects/s recovering 2024-09-13T05:39:17.918 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:17 smithi110 ceph-mon[49387]: pgmap v274: 97 pgs: 97 active+clean; 320 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 11 MiB/s wr, 300 op/s; 1.1 MiB/s, 24 keys/s, 13 objects/s recovering 2024-09-13T05:39:17.918 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:17 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T05:39:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:17 smithi053 ceph-mon[64292]: pgmap v274: 97 pgs: 97 active+clean; 320 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 11 MiB/s wr, 300 op/s; 1.1 MiB/s, 24 keys/s, 13 objects/s recovering 2024-09-13T05:39:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:17 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T05:39:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:18 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T05:39:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:18 smithi110 ceph-mon[49387]: Upgrade: osd.7 is safe to restart 2024-09-13T05:39:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T05:39:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:19.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:18 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T05:39:19.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:18 smithi053 ceph-mon[64292]: Upgrade: osd.7 is safe to restart 2024-09-13T05:39:19.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:19.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:19.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T05:39:19.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:20.041 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:19 smithi110 ceph-mon[49387]: pgmap v275: 97 pgs: 97 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 10 MiB/s wr, 311 op/s; 205 KiB/s, 7 objects/s recovering 2024-09-13T05:39:20.041 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:19 smithi110 ceph-mon[49387]: Upgrade: Updating osd.7 2024-09-13T05:39:20.042 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:19 smithi110 ceph-mon[49387]: Deploying daemon osd.7 on smithi110 2024-09-13T05:39:20.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:19 smithi053 ceph-mon[64292]: pgmap v275: 97 pgs: 97 active+clean; 283 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 10 MiB/s wr, 311 op/s; 205 KiB/s, 7 objects/s recovering 2024-09-13T05:39:20.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:19 smithi053 ceph-mon[64292]: Upgrade: Updating osd.7 2024-09-13T05:39:20.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:19 smithi053 ceph-mon[64292]: Deploying daemon osd.7 on smithi110 2024-09-13T05:39:21.782 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:21 smithi110 systemd[1]: Stopping Ceph osd.7 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:39:21.782 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:21 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[38983]: 2024-09-13T05:39:21.753+0000 7fb70258d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:39:21.782 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:21 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[38983]: 2024-09-13T05:39:21.753+0000 7fb70258d700 -1 osd.7 146 *** Got signal Terminated *** 2024-09-13T05:39:21.782 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:21 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[38983]: 2024-09-13T05:39:21.753+0000 7fb70258d700 -1 osd.7 146 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:39:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:21 smithi110 ceph-mon[49387]: pgmap v276: 97 pgs: 97 active+clean; 242 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 9.8 MiB/s wr, 295 op/s; 0 B/s, 1 objects/s recovering 2024-09-13T05:39:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:21 smithi110 ceph-mon[49387]: osd.7 marked itself down and dead 2024-09-13T05:39:22.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:21 smithi053 ceph-mon[64292]: pgmap v276: 97 pgs: 97 active+clean; 242 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 9.8 MiB/s wr, 295 op/s; 0 B/s, 1 objects/s recovering 2024-09-13T05:39:22.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:21 smithi053 ceph-mon[64292]: osd.7 marked itself down and dead 2024-09-13T05:39:22.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:22 smithi110 podman[58925]: 2024-09-13 05:39:22.489554049 +0000 UTC m=+0.835454164 container died 629528ca2e7600fec0a006f1171823ac163aa27ba6d66bb51293ce33b12f1404 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, architecture=x86_64, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-type=git) 2024-09-13T05:39:23.064 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:22 smithi110 ceph-mon[49387]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:39:23.065 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:22 smithi110 ceph-mon[49387]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T05:39:23.065 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:22 smithi110 ceph-mon[49387]: osdmap e147: 8 total, 7 up, 8 in 2024-09-13T05:39:23.065 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:22 smithi110 podman[58925]: 2024-09-13 05:39:22.829729524 +0000 UTC m=+1.175629642 container cleanup 629528ca2e7600fec0a006f1171823ac163aa27ba6d66bb51293ce33b12f1404 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.openshift.tags=base centos centos-stream, version=8, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_CLEAN=True, ceph=True, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T05:39:23.065 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:22 smithi110 bash[58925]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7 2024-09-13T05:39:23.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:22 smithi053 ceph-mon[64292]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T05:39:23.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:22 smithi053 ceph-mon[64292]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T05:39:23.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:22 smithi053 ceph-mon[64292]: osdmap e147: 8 total, 7 up, 8 in 2024-09-13T05:39:23.365 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:23 smithi110 podman[58943]: 2024-09-13 05:39:23.064562084 +0000 UTC m=+0.574382777 container remove 629528ca2e7600fec0a006f1171823ac163aa27ba6d66bb51293ce33b12f1404 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vcs-type=git) 2024-09-13T05:39:23.651 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:23 smithi110 podman[59030]: 2024-09-13 05:39:23.365524201 +0000 UTC m=+0.101398530 container create 3b696123fe8928614884687970b385da9d83f01bd58dac16b2711ebec5a8fca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T05:39:23.651 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:23 smithi110 podman[59030]: 2024-09-13 05:39:23.279979644 +0000 UTC m=+0.015853983 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:39:23.652 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:23 smithi110 podman[59030]: 2024-09-13 05:39:23.576588672 +0000 UTC m=+0.312463002 container init 3b696123fe8928614884687970b385da9d83f01bd58dac16b2711ebec5a8fca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True) 2024-09-13T05:39:23.652 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:23 smithi110 podman[59030]: 2024-09-13 05:39:23.579955952 +0000 UTC m=+0.315830282 container start 3b696123fe8928614884687970b385da9d83f01bd58dac16b2711ebec5a8fca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T05:39:23.652 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:23 smithi110 podman[59030]: 2024-09-13 05:39:23.651328983 +0000 UTC m=+0.387203321 container attach 3b696123fe8928614884687970b385da9d83f01bd58dac16b2711ebec5a8fca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T05:39:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:23 smithi110 ceph-mon[49387]: pgmap v278: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 236 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.8 MiB/s wr, 140 op/s 2024-09-13T05:39:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:23 smithi110 ceph-mon[49387]: osdmap e148: 8 total, 7 up, 8 in 2024-09-13T05:39:23.994 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:23 smithi110 podman[59030]: 2024-09-13 05:39:23.768841101 +0000 UTC m=+0.504715449 container died 3b696123fe8928614884687970b385da9d83f01bd58dac16b2711ebec5a8fca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-09-13T05:39:24.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:23 smithi053 ceph-mon[64292]: pgmap v278: 97 pgs: 8 peering, 12 stale+active+clean, 77 active+clean; 236 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.8 MiB/s wr, 140 op/s 2024-09-13T05:39:24.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:23 smithi053 ceph-mon[64292]: osdmap e148: 8 total, 7 up, 8 in 2024-09-13T05:39:24.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:24 smithi110 podman[59053]: 2024-09-13 05:39:24.330258608 +0000 UTC m=+0.555642940 container remove 3b696123fe8928614884687970b385da9d83f01bd58dac16b2711ebec5a8fca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T05:39:24.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:24 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-13T05:39:24.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:24 smithi110 systemd[1]: Stopped Ceph osd.7 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:39:24.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:24 smithi110 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 34.876s CPU time. 2024-09-13T05:39:24.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:24 smithi110 systemd[1]: Starting Ceph osd.7 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:39:25.001 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:24 smithi110 podman[59151]: 2024-09-13 05:39:24.748103633 +0000 UTC m=+0.107507001 container create dba2fd86edd317a89c5095c5fd0ca935ca6a118233c788e5c138e0f66ede9db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T05:39:25.001 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:24 smithi110 podman[59151]: 2024-09-13 05:39:24.656349576 +0000 UTC m=+0.015752888 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:39:25.001 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:25 smithi110 podman[59151]: 2024-09-13 05:39:25.000864003 +0000 UTC m=+0.360267311 container init dba2fd86edd317a89c5095c5fd0ca935ca6a118233c788e5c138e0f66ede9db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-09-13T05:39:25.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:25 smithi110 podman[59151]: 2024-09-13 05:39:25.004114443 +0000 UTC m=+0.363517755 container start dba2fd86edd317a89c5095c5fd0ca935ca6a118233c788e5c138e0f66ede9db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0) 2024-09-13T05:39:25.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:25 smithi110 podman[59151]: 2024-09-13 05:39:25.100844582 +0000 UTC m=+0.460247896 container attach dba2fd86edd317a89c5095c5fd0ca935ca6a118233c788e5c138e0f66ede9db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T05:39:25.495 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:25 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:39:25.495 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:25 smithi110 bash[59151]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:39:25.495 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:25 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:39:25.495 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:25 smithi110 bash[59151]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T05:39:26.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:25 smithi110 ceph-mon[49387]: pgmap v280: 97 pgs: 16 peering, 7 stale+active+clean, 74 active+clean; 234 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 838 KiB/s wr, 104 op/s 2024-09-13T05:39:26.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:25 smithi053 ceph-mon[64292]: pgmap v280: 97 pgs: 16 peering, 7 stale+active+clean, 74 active+clean; 234 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 838 KiB/s wr, 104 op/s 2024-09-13T05:39:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:26 smithi110 ceph-mon[49387]: pgmap v281: 97 pgs: 9 active+undersized, 16 peering, 2 stale+active+clean, 9 active+undersized+degraded, 61 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s wr, 16 op/s; 34/435 objects degraded (7.816%) 2024-09-13T05:39:27.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:26 smithi110 ceph-mon[49387]: Health check failed: Degraded data redundancy: 34/435 objects degraded (7.816%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:27.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:26 smithi053 ceph-mon[64292]: pgmap v281: 97 pgs: 9 active+undersized, 16 peering, 2 stale+active+clean, 9 active+undersized+degraded, 61 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s wr, 16 op/s; 34/435 objects degraded (7.816%) 2024-09-13T05:39:27.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:26 smithi053 ceph-mon[64292]: Health check failed: Degraded data redundancy: 34/435 objects degraded (7.816%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:27.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T05:39:27.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 bash[59151]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T05:39:27.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T05:39:27.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 bash[59151]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 bash[59151]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 bash[59151]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 bash[59151]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 bash[59151]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T05:39:28.096 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate[59166]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T05:39:28.097 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 bash[59151]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T05:39:28.097 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:27 smithi110 podman[59151]: 2024-09-13 05:39:27.804797688 +0000 UTC m=+3.164200998 container died dba2fd86edd317a89c5095c5fd0ca935ca6a118233c788e5c138e0f66ede9db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T05:39:28.842 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:28 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:28.842 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:28 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:28.842 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:28 smithi110 podman[59331]: 2024-09-13 05:39:28.629819924 +0000 UTC m=+0.814732915 container remove dba2fd86edd317a89c5095c5fd0ca935ca6a118233c788e5c138e0f66ede9db8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-09-13T05:39:28.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:28 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:28.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:28 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:29.094 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:28 smithi110 podman[59387]: 2024-09-13 05:39:28.745004519 +0000 UTC m=+0.016146408 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:39:29.094 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:28 smithi110 podman[59387]: 2024-09-13 05:39:28.880939508 +0000 UTC m=+0.152081442 container create 9afa18d580fe142d289b94e76b987fc902c886767fb9c5f8c14f1fd327ba760b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T05:39:29.094 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:29 smithi110 podman[59387]: 2024-09-13 05:39:29.091295898 +0000 UTC m=+0.362437786 container init 9afa18d580fe142d289b94e76b987fc902c886767fb9c5f8c14f1fd327ba760b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:39:29.095 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:29 smithi110 podman[59387]: 2024-09-13 05:39:29.09443856 +0000 UTC m=+0.365580456 container start 9afa18d580fe142d289b94e76b987fc902c886767fb9c5f8c14f1fd327ba760b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T05:39:29.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:29 smithi110 ceph-osd[59406]: -- 172.21.15.110:0/3142596664 <== mon.1 v2:172.21.15.110:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5621e311e1e0 con 0x5621e231fc00 2024-09-13T05:39:29.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:29 smithi110 bash[59387]: 9afa18d580fe142d289b94e76b987fc902c886767fb9c5f8c14f1fd327ba760b 2024-09-13T05:39:29.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:29 smithi110 systemd[1]: Started Ceph osd.7 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:39:29.958 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:29 smithi110 ceph-mon[49387]: pgmap v282: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s wr, 4 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:29.958 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:29.958 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:29.958 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:29 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:29.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:29 smithi053 ceph-mon[64292]: pgmap v282: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 37 KiB/s wr, 4 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:29.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:29.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:29.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:29 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:30.848 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:30 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:30.749+0000 7f27f0ec9640 -1 Falling back to public interface 2024-09-13T05:39:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:31 smithi110 ceph-mon[49387]: pgmap v283: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s wr, 2 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:31.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:31 smithi053 ceph-mon[64292]: pgmap v283: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s wr, 2 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:32 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 75/435 objects degraded (17.241%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:32.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:32 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 75/435 objects degraded (17.241%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:33.902 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:33 smithi110 ceph-mon[49387]: pgmap v284: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s wr, 1 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:33.902 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:33.902 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:33.902 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:33.902 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:33 smithi053 ceph-mon[64292]: pgmap v284: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s wr, 1 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:34.075 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:34.075 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:34.075 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:34.076 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:35.854 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:35 smithi110 ceph-mon[49387]: pgmap v285: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s wr, 1 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:35.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:35 smithi053 ceph-mon[64292]: pgmap v285: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s wr, 1 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:37.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:37.440+0000 7f27f0ec9640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: pgmap v286: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s wr, 1 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all osd 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-13T05:39:37.496 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T05:39:37.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-13T05:39:37.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T05:39:37.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-13T05:39:37.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T05:39:37.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T05:39:37.497 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: pgmap v286: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s wr, 1 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all osd 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-13T05:39:37.499 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T05:39:37.500 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-13T05:39:38.387 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:38 smithi053 ceph-mon[64292]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T05:39:38.387 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T05:39:38.387 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:38 smithi053 ceph-mon[64292]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T05:39:38.387 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:38 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi053.rzvrng"]} : dispatch 2024-09-13T05:39:38.387 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:38 smithi053 ceph-mon[64292]: Upgrade: It appears safe to stop mds.foofs.smithi053.rzvrng 2024-09-13T05:39:38.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:38 smithi110 ceph-mon[49387]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T05:39:38.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T05:39:38.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:38 smithi110 ceph-mon[49387]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T05:39:38.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:38 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi053.rzvrng"]} : dispatch 2024-09-13T05:39:38.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:38 smithi110 ceph-mon[49387]: Upgrade: It appears safe to stop mds.foofs.smithi053.rzvrng 2024-09-13T05:39:39.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:38 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:38.775+0000 7f27f0ec9640 -1 osd.7 146 log_to_monitors true 2024-09-13T05:39:39.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:39 smithi053 ceph-mon[64292]: pgmap v288: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 614 B/s wr, 0 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:39.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:39 smithi053 ceph-mon[64292]: from='osd.7 [v2:172.21.15.110:6824/3310387546,v1:172.21.15.110:6825/3310387546]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T05:39:39.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:39 smithi053 ceph-mon[64292]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T05:39:39.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:39.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:39:39.907 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:40.048 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:39 smithi110 ceph-mon[49387]: pgmap v288: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 614 B/s wr, 0 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:40.048 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:39 smithi110 ceph-mon[49387]: from='osd.7 [v2:172.21.15.110:6824/3310387546,v1:172.21.15.110:6825/3310387546]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T05:39:40.048 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:39 smithi110 ceph-mon[49387]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T05:39:40.048 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:40.048 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi053.rzvrng", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:39:40.048 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:39:40.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:40.047+0000 7f27e4c6b640 -1 osd.7 146 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T05:39:41.216 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:40 smithi053 ceph-mon[64292]: Upgrade: Updating mds.foofs.smithi053.rzvrng 2024-09-13T05:39:41.216 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:40 smithi053 ceph-mon[64292]: Deploying daemon mds.foofs.smithi053.rzvrng on smithi053 2024-09-13T05:39:41.216 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:40 smithi053 ceph-mon[64292]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T05:39:41.216 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:40 smithi053 ceph-mon[64292]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T05:39:41.217 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:40 smithi053 ceph-mon[64292]: from='osd.7 [v2:172.21.15.110:6824/3310387546,v1:172.21.15.110:6825/3310387546]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:41.217 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:40 smithi053 ceph-mon[64292]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:41.217 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:40 smithi053 ceph-mon[64292]: from='osd.7 ' entity='osd.7' 2024-09-13T05:39:41.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-mon[49387]: Upgrade: Updating mds.foofs.smithi053.rzvrng 2024-09-13T05:39:41.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-mon[49387]: Deploying daemon mds.foofs.smithi053.rzvrng on smithi053 2024-09-13T05:39:41.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-mon[49387]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T05:39:41.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-mon[49387]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T05:39:41.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-mon[49387]: from='osd.7 [v2:172.21.15.110:6824/3310387546,v1:172.21.15.110:6825/3310387546]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:41.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-mon[49387]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi110", "root=default"]} : dispatch 2024-09-13T05:39:41.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:40 smithi110 ceph-mon[49387]: from='osd.7 ' entity='osd.7' 2024-09-13T05:39:41.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:41 smithi053 ceph-mon[64292]: pgmap v290: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s wr, 0 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:41.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:41 smithi053 ceph-mon[64292]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:39:41.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:41 smithi053 ceph-mon[64292]: osd.7 [v2:172.21.15.110:6824/3310387546,v1:172.21.15.110:6825/3310387546] boot 2024-09-13T05:39:41.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:41 smithi053 ceph-mon[64292]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T05:39:41.822 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:41 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T05:39:42.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:41.881+0000 7f27cd422640 -1 bdev(0x5621e316ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 2 2024-09-13T05:39:42.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:41.882+0000 7f27d1c2b640 -1 bdev(0x5621e316ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 4 2024-09-13T05:39:42.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:41.883+0000 7f27ccc21640 -1 bdev(0x5621e316ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 4 2024-09-13T05:39:42.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:39:41.886+0000 7f27cf426640 -1 bdev(0x5621e316ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 5 2024-09-13T05:39:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-mon[49387]: pgmap v290: 97 pgs: 20 active+undersized, 22 active+undersized+degraded, 55 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s wr, 0 op/s; 75/435 objects degraded (17.241%) 2024-09-13T05:39:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-mon[49387]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T05:39:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-mon[49387]: osd.7 [v2:172.21.15.110:6824/3310387546,v1:172.21.15.110:6825/3310387546] boot 2024-09-13T05:39:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-mon[49387]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T05:39:42.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:41 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T05:39:42.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:41 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[64260]: 2024-09-13T05:39:41.820+0000 7fe096c02640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: Standby daemon mds.foofs.smithi110.akbygi assigned to filesystem foofs as rank 0 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T05:39:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi110.akbygi=up:replay} 2024-09-13T05:39:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:43.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: Standby daemon mds.foofs.smithi110.akbygi assigned to filesystem foofs as rank 0 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T05:39:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: fsmap foofs:1/1 {0=foofs.smithi110.akbygi=up:replay} 2024-09-13T05:39:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:43.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:43 smithi110 ceph-mon[49387]: pgmap v293: 97 pgs: 3 activating, 6 peering, 16 active+undersized, 13 active+undersized+degraded, 59 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 472 B/s rd, 0 op/s; 50/435 objects degraded (11.494%) 2024-09-13T05:39:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:43 smithi110 ceph-mon[49387]: Health check update: Degraded data redundancy: 50/435 objects degraded (11.494%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:43 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:39:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:43 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:reconnect 2024-09-13T05:39:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:43 smithi110 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi110.akbygi=up:reconnect} 2024-09-13T05:39:44.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:43 smithi053 ceph-mon[64292]: pgmap v293: 97 pgs: 3 activating, 6 peering, 16 active+undersized, 13 active+undersized+degraded, 59 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 472 B/s rd, 0 op/s; 50/435 objects degraded (11.494%) 2024-09-13T05:39:44.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:43 smithi053 ceph-mon[64292]: Health check update: Degraded data redundancy: 50/435 objects degraded (11.494%), 13 pgs degraded (PG_DEGRADED) 2024-09-13T05:39:44.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:43 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:39:44.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:43 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:reconnect 2024-09-13T05:39:44.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:43 smithi053 ceph-mon[64292]: fsmap foofs:1/1 {0=foofs.smithi110.akbygi=up:reconnect} 2024-09-13T05:39:44.885 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:39:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:44 smithi110 ceph-mon[49387]: reconnect by client.24375 172.21.15.110:0/1388832170 after 0.000999983 2024-09-13T05:39:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:44 smithi110 ceph-mon[49387]: reconnect by client.14564 172.21.15.53:0/1940818184 after 0.0539991 2024-09-13T05:39:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:44 smithi110 ceph-mon[49387]: pgmap v294: 97 pgs: 3 activating, 6 peering, 14 active+undersized, 12 active+undersized+degraded, 62 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-09-13T05:39:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:44 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:rejoin 2024-09-13T05:39:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:44 smithi110 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi110.akbygi=up:rejoin} 2024-09-13T05:39:45.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:44 smithi110 ceph-mon[49387]: daemon mds.foofs.smithi110.akbygi is now active in filesystem foofs as rank 0 2024-09-13T05:39:45.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:44 smithi053 ceph-mon[64292]: reconnect by client.24375 172.21.15.110:0/1388832170 after 0.000999983 2024-09-13T05:39:45.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:44 smithi053 ceph-mon[64292]: reconnect by client.14564 172.21.15.53:0/1940818184 after 0.0539991 2024-09-13T05:39:45.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:44 smithi053 ceph-mon[64292]: pgmap v294: 97 pgs: 3 activating, 6 peering, 14 active+undersized, 12 active+undersized+degraded, 62 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 48/435 objects degraded (11.034%) 2024-09-13T05:39:45.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:44 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:rejoin 2024-09-13T05:39:45.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:44 smithi053 ceph-mon[64292]: fsmap foofs:1/1 {0=foofs.smithi110.akbygi=up:rejoin} 2024-09-13T05:39:45.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:44 smithi053 ceph-mon[64292]: daemon mds.foofs.smithi110.akbygi is now active in filesystem foofs as rank 0 2024-09-13T05:39:45.598 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (9m) 2m ago 19m 18.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (5m) 2m ago 19m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (5m) 12s ago 18m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (9m) 2m ago 18m 44.3M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (13m) 2m ago 13m 38.1M - 17.2.0 e1d6a67b021e 0ea9005535f8 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (13m) 12s ago 13m 17.0M - 17.2.0 e1d6a67b021e b8226fecb6eb 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (8m) 2m ago 20m 518M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (7m) 12s ago 18m 456M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (7m) 2m ago 20m 52.1M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (6m) 12s ago 18m 47.2M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (12m) 2m ago 12m 279M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (12m) 12s ago 12m 90.9M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (8m) 2m ago 18m 10.2M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:39:45.599 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (8m) 12s ago 18m 11.1M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (5m) 2m ago 17m 307M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (4m) 2m ago 17m 248M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (3m) 2m ago 16m 195M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (2m) 2m ago 16m 14.2M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (2m) 12s ago 15m 318M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (91s) 12s ago 15m 269M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 caa8bc291506 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (53s) 12s ago 15m 237M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 260b18297eb5 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (16s) 12s ago 14m 12.1M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 9afa18d580fe 2024-09-13T05:39:45.600 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (7m) 2m ago 18m 54.0M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:39:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:45 smithi053 ceph-mon[64292]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:45 smithi053 ceph-mon[64292]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:45 smithi053 ceph-mon[64292]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T05:39:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:45 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:active 2024-09-13T05:39:45.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:45 smithi053 ceph-mon[64292]: fsmap foofs:1 {0=foofs.smithi110.akbygi=up:active} 2024-09-13T05:39:45.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:45 smithi053 ceph-mon[64292]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:46.029 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:39:46.029 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:39:46.029 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 8 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T05:39:46.030 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 12 2024-09-13T05:39:46.031 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:39:46.031 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:39:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:45 smithi110 ceph-mon[49387]: from='client.34438 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:45 smithi110 ceph-mon[49387]: from='client.34442 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:45 smithi110 ceph-mon[49387]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T05:39:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:45 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.110:6832/724929159,v1:172.21.15.110:6833/724929159] up:active 2024-09-13T05:39:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:45 smithi110 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi110.akbygi=up:active} 2024-09-13T05:39:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:45 smithi110 ceph-mon[49387]: from='client.34446 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:46.390 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:39:46.390 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:39:46.390 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:39:46.390 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:39:46.390 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: "osd", 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "14/23 daemons upgraded", 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading mds daemons", 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:39:46.391 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:39:46.825 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_WARN insufficient standby MDS daemons available 2024-09-13T05:39:46.825 INFO:teuthology.orchestra.run.smithi053.stdout:[WRN] MDS_INSUFFICIENT_STANDBY: insufficient standby MDS daemons available 2024-09-13T05:39:46.825 INFO:teuthology.orchestra.run.smithi053.stdout: have 0; want 1 more 2024-09-13T05:39:47.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:46 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1438740499' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:47.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:46 smithi110 ceph-mon[49387]: pgmap v295: 97 pgs: 3 activating, 6 peering, 88 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 7 op/s 2024-09-13T05:39:47.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:46 smithi110 ceph-mon[49387]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:47.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:46 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1438740499' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:39:47.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:46 smithi053 ceph-mon[64292]: pgmap v295: 97 pgs: 3 activating, 6 peering, 88 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 7 op/s 2024-09-13T05:39:47.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:46 smithi053 ceph-mon[64292]: from='client.34454 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:39:48.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:47 smithi110 ceph-mon[49387]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/435 objects degraded (11.034%), 12 pgs degraded) 2024-09-13T05:39:48.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:47 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/909256864' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:39:48.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:47 smithi053 ceph-mon[64292]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 48/435 objects degraded (11.034%), 12 pgs degraded) 2024-09-13T05:39:48.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:47 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/909256864' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:39:49.145 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:48 smithi110 ceph-mon[49387]: pgmap v296: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 255 B/s wr, 11 op/s 2024-09-13T05:39:49.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:48 smithi053 ceph-mon[64292]: pgmap v296: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 255 B/s wr, 11 op/s 2024-09-13T05:39:51.493 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:51 smithi053 ceph-mon[64292]: pgmap v297: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 226 B/s wr, 12 op/s 2024-09-13T05:39:51.493 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:51 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:51.493 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:51 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:51.493 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:51 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:51.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:51 smithi110 ceph-mon[49387]: pgmap v297: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 226 B/s wr, 12 op/s 2024-09-13T05:39:51.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:51 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:51.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:51 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:51.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:51 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:39:52.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:52 smithi053 ceph-mon[64292]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T05:39:52.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:52 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:39:52.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:52 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:boot 2024-09-13T05:39:52.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:52 smithi053 ceph-mon[64292]: fsmap foofs:1 {0=foofs.smithi110.akbygi=up:active} 1 up:standby 2024-09-13T05:39:52.461 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:52 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:39:52.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:52 smithi110 ceph-mon[49387]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T05:39:52.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:52 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:39:52.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:52 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:boot 2024-09-13T05:39:52.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:52 smithi110 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi110.akbygi=up:active} 1 up:standby 2024-09-13T05:39:52.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:52 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:39:53.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:53 smithi110 ceph-mon[49387]: pgmap v298: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.4 KiB/s wr, 14 op/s 2024-09-13T05:39:53.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:53 smithi053 ceph-mon[64292]: pgmap v298: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.4 KiB/s wr, 14 op/s 2024-09-13T05:39:55.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:55 smithi110 ceph-mon[49387]: pgmap v299: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T05:39:55.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:55 smithi053 ceph-mon[64292]: pgmap v299: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T05:39:57.174 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:56 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:57.175 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:56 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:57.175 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:56 smithi053 ceph-mon[64292]: pgmap v300: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T05:39:57.175 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:56 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:57.175 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:56 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:57.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:56 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:57.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:56 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:57.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:56 smithi110 ceph-mon[49387]: pgmap v300: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T05:39:57.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:56 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:57.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:56 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:58.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:58.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:58.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:58 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:39:58.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:58 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:39:59.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:39:59 smithi053 ceph-mon[64292]: pgmap v301: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-13T05:40:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:39:59 smithi110 ceph-mon[49387]: pgmap v301: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 7.2 KiB/s wr, 8 op/s 2024-09-13T05:40:01.082 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: overall HEALTH_OK 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.083 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.084 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi110.akbygi"]} : dispatch 2024-09-13T05:40:01.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: overall HEALTH_OK 2024-09-13T05:40:01.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:01.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi110.akbygi"]} : dispatch 2024-09-13T05:40:02.147 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:01 smithi110 ceph-mon[49387]: Detected new or changed devices on smithi053 2024-09-13T05:40:02.148 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:01 smithi110 ceph-mon[49387]: pgmap v302: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 7.0 KiB/s wr, 4 op/s 2024-09-13T05:40:02.148 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:01 smithi110 ceph-mon[49387]: Upgrade: It appears safe to stop mds.foofs.smithi110.akbygi 2024-09-13T05:40:02.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:01 smithi053 ceph-mon[64292]: Detected new or changed devices on smithi053 2024-09-13T05:40:02.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:01 smithi053 ceph-mon[64292]: pgmap v302: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 7.0 KiB/s wr, 4 op/s 2024-09-13T05:40:02.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:01 smithi053 ceph-mon[64292]: Upgrade: It appears safe to stop mds.foofs.smithi110.akbygi 2024-09-13T05:40:03.134 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:02 smithi110 ceph-mon[49387]: Upgrade: Updating mds.foofs.smithi110.akbygi 2024-09-13T05:40:03.134 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:03.134 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:40:03.134 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:02 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:03.135 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:02 smithi110 ceph-mon[49387]: Deploying daemon mds.foofs.smithi110.akbygi on smithi110 2024-09-13T05:40:03.135 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:02 smithi110 ceph-mon[49387]: pgmap v303: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-09-13T05:40:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:02 smithi053 ceph-mon[64292]: Upgrade: Updating mds.foofs.smithi110.akbygi 2024-09-13T05:40:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi110.akbygi", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T05:40:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:02 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:02 smithi053 ceph-mon[64292]: Deploying daemon mds.foofs.smithi110.akbygi on smithi110 2024-09-13T05:40:03.248 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:02 smithi053 ceph-mon[64292]: pgmap v303: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 7.2 KiB/s wr, 3 op/s 2024-09-13T05:40:03.994 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:40:03.611+0000 7f27d1c2b640 -1 bdev(0x5621e316ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 2 2024-09-13T05:40:03.995 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:40:03.611+0000 7f27ccc21640 -1 bdev(0x5621e316ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 3 2024-09-13T05:40:03.995 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:40:03.612+0000 7f27cf426640 -1 bdev(0x5621e316ec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 4 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: Standby daemon mds.foofs.smithi053.rzvrng assigned to filesystem foofs as rank 0 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T05:40:03.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:03 smithi110 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi053.rzvrng=up:replay} 2024-09-13T05:40:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[64260]: 2024-09-13T05:40:03.589+0000 7fe096c02640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:40:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T05:40:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T05:40:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T05:40:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: Standby daemon mds.foofs.smithi053.rzvrng assigned to filesystem foofs as rank 0 2024-09-13T05:40:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T05:40:03.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T05:40:03.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T05:40:03.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:03 smithi053 ceph-mon[64292]: fsmap foofs:1/1 {0=foofs.smithi053.rzvrng=up:replay} 2024-09-13T05:40:05.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:04 smithi110 ceph-mon[49387]: pgmap v305: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s 2024-09-13T05:40:05.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:04 smithi053 ceph-mon[64292]: pgmap v305: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 307 B/s wr, 0 op/s 2024-09-13T05:40:06.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:05 smithi110 ceph-mon[49387]: reconnect by client.14564 172.21.15.53:0/1940818184 after 0.000999982 2024-09-13T05:40:06.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:05 smithi110 ceph-mon[49387]: reconnect by client.24375 172.21.15.110:0/1388832170 after 0.000999982 2024-09-13T05:40:06.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:05 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:reconnect 2024-09-13T05:40:06.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:05 smithi110 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi053.rzvrng=up:reconnect} 2024-09-13T05:40:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:05 smithi053 ceph-mon[64292]: reconnect by client.14564 172.21.15.53:0/1940818184 after 0.000999982 2024-09-13T05:40:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:05 smithi053 ceph-mon[64292]: reconnect by client.24375 172.21.15.110:0/1388832170 after 0.000999982 2024-09-13T05:40:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:05 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:reconnect 2024-09-13T05:40:06.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:05 smithi053 ceph-mon[64292]: fsmap foofs:1/1 {0=foofs.smithi053.rzvrng=up:reconnect} 2024-09-13T05:40:07.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:06 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:rejoin 2024-09-13T05:40:07.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:06 smithi110 ceph-mon[49387]: fsmap foofs:1/1 {0=foofs.smithi053.rzvrng=up:rejoin} 2024-09-13T05:40:07.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:06 smithi110 ceph-mon[49387]: daemon mds.foofs.smithi053.rzvrng is now active in filesystem foofs as rank 0 2024-09-13T05:40:07.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:06 smithi110 ceph-mon[49387]: pgmap v306: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 307 B/s wr, 4 op/s 2024-09-13T05:40:07.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:06 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:rejoin 2024-09-13T05:40:07.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:06 smithi053 ceph-mon[64292]: fsmap foofs:1/1 {0=foofs.smithi053.rzvrng=up:rejoin} 2024-09-13T05:40:07.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:06 smithi053 ceph-mon[64292]: daemon mds.foofs.smithi053.rzvrng is now active in filesystem foofs as rank 0 2024-09-13T05:40:07.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:06 smithi053 ceph-mon[64292]: pgmap v306: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 307 B/s wr, 4 op/s 2024-09-13T05:40:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:07 smithi110 ceph-mon[49387]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T05:40:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:07 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:active 2024-09-13T05:40:08.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:07 smithi110 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 2024-09-13T05:40:08.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:07 smithi053 ceph-mon[64292]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T05:40:08.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:07 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086] up:active 2024-09-13T05:40:08.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:07 smithi053 ceph-mon[64292]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 2024-09-13T05:40:09.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:08 smithi110 ceph-mon[49387]: pgmap v307: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T05:40:09.973 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:08 smithi053 ceph-mon[64292]: pgmap v307: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 511 B/s wr, 7 op/s 2024-09-13T05:40:11.240 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:11 smithi110 ceph-mon[49387]: pgmap v308: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 511 B/s wr, 9 op/s 2024-09-13T05:40:11.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:11 smithi053 ceph-mon[64292]: pgmap v308: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 511 B/s wr, 9 op/s 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: Cluster is now healthy 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: mds.? [v2:172.21.15.110:6832/2036249896,v1:172.21.15.110:6833/2036249896] up:boot 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"} : dispatch 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:12.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:12.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T05:40:12.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: Cluster is now healthy 2024-09-13T05:40:12.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: mds.? [v2:172.21.15.110:6832/2036249896,v1:172.21.15.110:6833/2036249896] up:boot 2024-09-13T05:40:12.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: fsmap foofs:1 {0=foofs.smithi053.rzvrng=up:active} 1 up:standby 2024-09-13T05:40:12.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "mds metadata", "who": "foofs.smithi110.akbygi"} : dispatch 2024-09-13T05:40:12.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:12.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:12.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:13.633 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:13 smithi110 ceph-mon[49387]: pgmap v309: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-13T05:40:13.633 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:13.633 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:13.633 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:13 smithi053 ceph-mon[64292]: pgmap v309: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.7 KiB/s wr, 14 op/s 2024-09-13T05:40:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:13.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:15 smithi110 ceph-mon[49387]: pgmap v310: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.3 KiB/s wr, 14 op/s 2024-09-13T05:40:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:15.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:15 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:15.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:15 smithi053 ceph-mon[64292]: pgmap v310: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 8.3 KiB/s wr, 14 op/s 2024-09-13T05:40:15.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:15.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:15 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:16 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:16.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:16 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:16.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:16 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:16.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:16 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:17.217 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:40:17.929 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:40:17.929 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (9m) 22s ago 19m 20.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:40:17.929 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (6m) 22s ago 19m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:40:17.930 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (6m) 2s ago 19m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:40:17.930 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (9m) 22s ago 19m 50.4M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:40:17.930 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (27s) 22s ago 13m 14.1M - 19.3.0-4911-g9d608eb1 0ce38589d192 c5ed825917f6 2024-09-13T05:40:17.930 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (6s) 2s ago 13m 12.9M - 19.3.0-4911-g9d608eb1 0ce38589d192 03edcbdbc507 2024-09-13T05:40:17.930 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (8m) 22s ago 21m 522M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:40:17.930 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (8m) 2s ago 19m 456M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:40:17.931 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (7m) 22s ago 21m 60.8M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:40:17.931 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (7m) 2s ago 19m 48.3M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:40:17.931 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (13m) 22s ago 13m 267M - 4.0 e1d6a67b021e 1554adefe06a 2024-09-13T05:40:17.931 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (13m) 2s ago 13m 90.9M - 4.0 e1d6a67b021e f51267c373e5 2024-09-13T05:40:17.931 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (9m) 22s ago 19m 9.95M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:40:17.931 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (9m) 2s ago 18m 11.6M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:40:17.931 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (5m) 22s ago 18m 326M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:40:17.932 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (5m) 22s ago 17m 362M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:40:17.932 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (4m) 22s ago 17m 313M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:40:17.932 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (3m) 22s ago 16m 335M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:40:17.932 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (2m) 2s ago 16m 318M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:40:17.932 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (2m) 2s ago 16m 270M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 caa8bc291506 2024-09-13T05:40:17.932 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (85s) 2s ago 15m 239M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 260b18297eb5 2024-09-13T05:40:17.933 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (49s) 2s ago 15m 269M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 9afa18d580fe 2024-09-13T05:40:17.933 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (8m) 22s ago 18m 55.0M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:40:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:17 smithi110 ceph-mon[49387]: pgmap v311: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T05:40:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:17 smithi053 ceph-mon[64292]: pgmap v311: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:18.364 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 8 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 14 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:40:18.365 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:40:18.743 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:40:18.743 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:40:18.743 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "osd", 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "mds", 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr" 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "16/23 daemons upgraded", 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading mds daemons", 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:40:18.744 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:40:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/954479940' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:18.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:40:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:18.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:18 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='client.34466 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/954479940' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:40:18.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:18.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:18 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.232 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:40:19.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: pgmap v312: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: Detected new or changed devices on smithi110 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi053.rzvrng"}]': finished 2024-09-13T05:40:19.748 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi110.akbygi"} : dispatch 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi110.akbygi"}]': finished 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.749 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:19 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1515166324' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:40:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: pgmap v312: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 8.5 MiB/s rd, 7.2 KiB/s wr, 9 op/s 2024-09-13T05:40:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: Detected new or changed devices on smithi110 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='client.34482 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi053.rzvrng"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi053.rzvrng"}]': finished 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi110.akbygi"} : dispatch 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi110.akbygi"}]': finished 2024-09-13T05:40:19.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:19.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:19 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1515166324' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:40:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:20 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all mds 2024-09-13T05:40:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:20 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all rgw 2024-09-13T05:40:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:20 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T05:40:20.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:20 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T05:40:20.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:20 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all iscsi 2024-09-13T05:40:20.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:20 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all mds 2024-09-13T05:40:20.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:20 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all rgw 2024-09-13T05:40:20.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:20 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T05:40:20.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:20 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T05:40:20.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:20 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all iscsi 2024-09-13T05:40:21.992 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: pgmap v313: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 7.1 KiB/s wr, 6 op/s 2024-09-13T05:40:21.992 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: Upgrade: Updating nfs.foo.0.0.smithi053.zidqnx (1/2) 2024-09-13T05:40:21.992 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:21.992 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx 2024-09-13T05:40:21.992 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T05:40:21.993 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:21 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: pgmap v313: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.6 KiB/s rd, 7.1 KiB/s wr, 6 op/s 2024-09-13T05:40:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: Upgrade: Updating nfs.foo.0.0.smithi053.zidqnx (1/2) 2024-09-13T05:40:22.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi053.zidqnx-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T05:40:22.245 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:21 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:22.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:22 smithi053 ceph-mon[64292]: Rados config object exists: conf-nfs.foo 2024-09-13T05:40:22.934 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:22 smithi053 ceph-mon[64292]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx-rgw 2024-09-13T05:40:22.935 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:22 smithi053 ceph-mon[64292]: Bind address in nfs.foo.0.0.smithi053.zidqnx's ganesha conf is defaulting to empty 2024-09-13T05:40:22.935 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:22 smithi053 ceph-mon[64292]: Deploying daemon nfs.foo.0.0.smithi053.zidqnx on smithi053 2024-09-13T05:40:23.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:22 smithi110 ceph-mon[49387]: Rados config object exists: conf-nfs.foo 2024-09-13T05:40:23.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:22 smithi110 ceph-mon[49387]: Creating key for client.nfs.foo.0.0.smithi053.zidqnx-rgw 2024-09-13T05:40:23.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:22 smithi110 ceph-mon[49387]: Bind address in nfs.foo.0.0.smithi053.zidqnx's ganesha conf is defaulting to empty 2024-09-13T05:40:23.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:22 smithi110 ceph-mon[49387]: Deploying daemon nfs.foo.0.0.smithi053.zidqnx on smithi053 2024-09-13T05:40:23.857 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:23 smithi053 ceph-mon[64292]: pgmap v314: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-13T05:40:24.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:23 smithi110 ceph-mon[49387]: pgmap v314: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-13T05:40:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:24 smithi110 ceph-mon[49387]: Evicting (and blocklisting) client session 14564 (172.21.15.53:0/1940818184) 2024-09-13T05:40:24.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:24 smithi110 ceph-mon[49387]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.53:0/1940818184"} : dispatch 2024-09-13T05:40:24.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:24 smithi053 ceph-mon[64292]: Evicting (and blocklisting) client session 14564 (172.21.15.53:0/1940818184) 2024-09-13T05:40:24.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:24 smithi053 ceph-mon[64292]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.53:0/1940818184"} : dispatch 2024-09-13T05:40:26.095 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:25 smithi110 ceph-mon[49387]: pgmap v315: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-13T05:40:26.096 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:25 smithi110 ceph-mon[49387]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.53:0/1940818184"}]': finished 2024-09-13T05:40:26.096 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:25 smithi110 ceph-mon[49387]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T05:40:26.096 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:25 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:26.096 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:25 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:26.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:25 smithi053 ceph-mon[64292]: pgmap v315: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-13T05:40:26.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:25 smithi053 ceph-mon[64292]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.53:0/1940818184"}]': finished 2024-09-13T05:40:26.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:25 smithi053 ceph-mon[64292]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T05:40:26.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:25 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:26.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:25 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: pgmap v317: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s 2024-09-13T05:40:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: Upgrade: Updating nfs.foo.1.0.smithi110.fvztpo (2/2) 2024-09-13T05:40:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo 2024-09-13T05:40:27.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: Rados config object exists: conf-nfs.foo 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo-rgw 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: Bind address in nfs.foo.1.0.smithi110.fvztpo's ganesha conf is defaulting to empty 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:27.495 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:27 smithi110 ceph-mon[49387]: Deploying daemon nfs.foo.1.0.smithi110.fvztpo on smithi110 2024-09-13T05:40:27.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: pgmap v317: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 409 B/s rd, 204 B/s wr, 0 op/s 2024-09-13T05:40:27.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: Upgrade: Updating nfs.foo.1.0.smithi110.fvztpo (2/2) 2024-09-13T05:40:27.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:27.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo 2024-09-13T05:40:27.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T05:40:27.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T05:40:27.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: Rados config object exists: conf-nfs.foo 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: Creating key for client.nfs.foo.1.0.smithi110.fvztpo-rgw 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi110.fvztpo-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: Bind address in nfs.foo.1.0.smithi110.fvztpo's ganesha conf is defaulting to empty 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:27.498 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:27 smithi053 ceph-mon[64292]: Deploying daemon nfs.foo.1.0.smithi110.fvztpo on smithi110 2024-09-13T05:40:28.374 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:28 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:28.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:28 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:29.379 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:29 smithi110 ceph-mon[49387]: pgmap v318: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:29.497 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:29 smithi053 ceph-mon[64292]: pgmap v318: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:30.953 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:30.953 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:30.953 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:30 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:30.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:30.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:30.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:30 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:31 smithi110 ceph-mon[49387]: pgmap v319: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:31.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:31 smithi053 ceph-mon[64292]: pgmap v319: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:32 smithi110 ceph-mon[49387]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.110:0/1388832170"} : dispatch 2024-09-13T05:40:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:32 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:32.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:32 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:32.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:32 smithi053 ceph-mon[64292]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.110:0/1388832170"} : dispatch 2024-09-13T05:40:32.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:32 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:32.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:32 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:33.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:33 smithi110 ceph-mon[49387]: pgmap v320: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-09-13T05:40:33.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:33 smithi110 ceph-mon[49387]: Evicting (and blocklisting) client session 24375 (172.21.15.110:0/1388832170) 2024-09-13T05:40:33.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:33 smithi110 ceph-mon[49387]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.110:0/1388832170"}]': finished 2024-09-13T05:40:33.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:33 smithi110 ceph-mon[49387]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T05:40:33.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:33.930 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:33 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:33 smithi053 ceph-mon[64292]: pgmap v320: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 1.9 KiB/s wr, 7 op/s 2024-09-13T05:40:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:33 smithi053 ceph-mon[64292]: Evicting (and blocklisting) client session 24375 (172.21.15.110:0/1388832170) 2024-09-13T05:40:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:33 smithi053 ceph-mon[64292]: from='mds.? [v2:172.21.15.53:6834/370131086,v1:172.21.15.53:6835/370131086]' entity='mds.foofs.smithi053.rzvrng' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.110:0/1388832170"}]': finished 2024-09-13T05:40:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:33 smithi053 ceph-mon[64292]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T05:40:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:33 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:35 smithi053 ceph-mon[64292]: pgmap v322: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-09-13T05:40:35.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.793 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:35 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:35 smithi110 ceph-mon[49387]: pgmap v322: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-09-13T05:40:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:35.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:35 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:37.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:37 smithi110 ceph-mon[49387]: pgmap v323: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-13T05:40:37.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:37.744 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:37 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:37.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:37 smithi053 ceph-mon[64292]: pgmap v323: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.0 KiB/s wr, 7 op/s 2024-09-13T05:40:37.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:37.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:37 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: pgmap v324: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:39.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.801 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.802 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi053.zidqnx"} : dispatch 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi053.zidqnx"}]': finished 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi110.fvztpo"} : dispatch 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi110.fvztpo"}]': finished 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.803 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:39 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: pgmap v324: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi053.zidqnx"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi053.zidqnx"}]': finished 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi110.fvztpo"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi110.fvztpo"}]': finished 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:39.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:39 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:40.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:40 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all nfs 2024-09-13T05:40:40.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:40 smithi110 ceph-mon[49387]: Upgrade: Setting container_image for all nvmeof 2024-09-13T05:40:40.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:40 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all nfs 2024-09-13T05:40:40.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:40 smithi053 ceph-mon[64292]: Upgrade: Setting container_image for all nvmeof 2024-09-13T05:40:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:41 smithi110 ceph-mon[49387]: pgmap v325: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:41 smithi110 ceph-mon[49387]: Upgrade: Updating grafana.smithi053 2024-09-13T05:40:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:41 smithi053 ceph-mon[64292]: pgmap v325: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T05:40:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:41 smithi053 ceph-mon[64292]: Upgrade: Updating grafana.smithi053 2024-09-13T05:40:42.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:42 smithi110 ceph-mon[49387]: Deploying daemon grafana.smithi053 on smithi053 2024-09-13T05:40:42.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:42.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:42 smithi053 ceph-mon[64292]: Deploying daemon grafana.smithi053 on smithi053 2024-09-13T05:40:42.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:43 smithi110 ceph-mon[49387]: pgmap v326: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 511 B/s wr, 0 op/s 2024-09-13T05:40:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:43 smithi053 ceph-mon[64292]: pgmap v326: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 511 B/s wr, 0 op/s 2024-09-13T05:40:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:45 smithi110 ceph-mon[49387]: pgmap v327: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 89 B/s rd, 358 B/s wr, 0 op/s 2024-09-13T05:40:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:45 smithi053 ceph-mon[64292]: pgmap v327: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 89 B/s rd, 358 B/s wr, 0 op/s 2024-09-13T05:40:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:47 smithi110 ceph-mon[49387]: pgmap v328: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:40:47.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:47 smithi053 ceph-mon[64292]: pgmap v328: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:40:49.623 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:40:49.898 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:49 smithi053 ceph-mon[64292]: pgmap v329: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:40:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:49 smithi110 ceph-mon[49387]: pgmap v329: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T05:40:50.350 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:40:50.350 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (10m) 15s ago 20m 20.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:40:50.350 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (7m) 15s ago 20m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:40:50.350 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (6m) 17s ago 19m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (10m) 15s ago 20m 51.5M - dad864ee21e9 6fb28c7faca9 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (59s) 15s ago 14m 39.9M - 19.3.0-4911-g9d608eb1 0ce38589d192 c5ed825917f6 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (38s) 17s ago 14m 13.0M - 19.3.0-4911-g9d608eb1 0ce38589d192 03edcbdbc507 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (9m) 15s ago 21m 523M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (8m) 17s ago 19m 457M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (8m) 15s ago 21m 62.6M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (7m) 17s ago 19m 49.6M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (26s) 15s ago 13m 66.7M - 5.9 0ce38589d192 a660bae25d54 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (21s) 17s ago 13m 19.7M - 5.9 0ce38589d192 7e2784544425 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (10m) 15s ago 19m 11.7M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (9m) 17s ago 19m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (6m) 15s ago 18m 328M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:40:50.351 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (5m) 15s ago 18m 363M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:40:50.352 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (4m) 15s ago 17m 315M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:40:50.352 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (3m) 15s ago 17m 337M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:40:50.352 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (3m) 17s ago 16m 318M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:40:50.352 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (2m) 17s ago 16m 270M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 caa8bc291506 2024-09-13T05:40:50.352 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (117s) 17s ago 16m 240M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 260b18297eb5 2024-09-13T05:40:50.352 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (81s) 17s ago 15m 269M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 9afa18d580fe 2024-09-13T05:40:50.352 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (8m) 15s ago 19m 55.2M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:40:50.787 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:40:50.787 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:40:50.787 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:40:50.787 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:50.787 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:40:50.787 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:40:50.787 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 8 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 14 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:40:50.788 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:40:50.972 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:50 smithi110 ceph-mon[49387]: from='client.34526 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:50.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:50 smithi053 ceph-mon[64292]: from='client.34526 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:51.154 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:40:51.154 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": true, 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [ 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "crash", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "osd", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "mon", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "mds", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "nfs" 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": "18/23 daemons upgraded", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:40:51.155 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:40:51.628 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:40:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:51 smithi110 ceph-mon[49387]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:51 smithi110 ceph-mon[49387]: pgmap v330: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:51 smithi110 ceph-mon[49387]: from='client.34534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:51 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/985284552' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:51 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3443751096' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:40:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:51 smithi053 ceph-mon[64292]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:51 smithi053 ceph-mon[64292]: pgmap v330: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:51 smithi053 ceph-mon[64292]: from='client.34534 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:51 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/985284552' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:40:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:51 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3443751096' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:40:52.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:52 smithi110 ceph-mon[49387]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:52.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:52 smithi053 ceph-mon[64292]: from='client.34542 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:40:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:53 smithi110 ceph-mon[49387]: pgmap v331: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:53.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:53 smithi053 ceph-mon[64292]: pgmap v331: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:55.827 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:55 smithi053 ceph-mon[64292]: pgmap v332: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:55.827 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:55 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:55.828 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:55 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:55.828 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:55 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:55 smithi110 ceph-mon[49387]: pgmap v332: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:55 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:55 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:40:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:55 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:40:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:57 smithi110 ceph-mon[49387]: pgmap v333: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:57.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:57 smithi053 ceph-mon[64292]: pgmap v333: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:40:57.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:40:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:40:59 smithi110 ceph-mon[49387]: pgmap v334: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:40:59.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:40:59 smithi053 ceph-mon[64292]: pgmap v334: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:01.158 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:01.159 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:01.159 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:00 smithi053 ceph-mon[64292]: pgmap v335: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:01.159 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:01.159 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:00 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:01.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:01.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:01.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:00 smithi110 ceph-mon[49387]: pgmap v335: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:01.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:01.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:00 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:03 smithi053 ceph-mon[64292]: pgmap v336: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:03.494 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:03 smithi110 ceph-mon[49387]: pgmap v336: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:04.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:04.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:04.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:41:04.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:41:04.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:04.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:04.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T05:41:04.998 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:04.999 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.000 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:04 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-13T05:41:05.001 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.002 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T05:41:05.003 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T05:41:05.004 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:41:05.005 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:41:05.006 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.006 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.006 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:41:05.006 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:41:05.006 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:41:05.006 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.006 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:04 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:05 smithi110 ceph-mon[49387]: pgmap v337: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:05 smithi110 ceph-mon[49387]: Upgrade: Finalizing container_image settings 2024-09-13T05:41:05.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:05 smithi110 ceph-mon[49387]: Upgrade: Complete! 2024-09-13T05:41:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:05 smithi053 ceph-mon[64292]: pgmap v337: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:05 smithi053 ceph-mon[64292]: Upgrade: Finalizing container_image settings 2024-09-13T05:41:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:05 smithi053 ceph-mon[64292]: Upgrade: Complete! 2024-09-13T05:41:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:07 smithi110 ceph-mon[49387]: pgmap v338: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:07.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:07 smithi053 ceph-mon[64292]: pgmap v338: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:09.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:08 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:09.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:08 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:41:10.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:09 smithi110 ceph-mon[49387]: pgmap v339: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:10.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:09 smithi053 ceph-mon[64292]: pgmap v339: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:12.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:11 smithi110 ceph-mon[49387]: pgmap v340: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:12.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:11 smithi053 ceph-mon[64292]: pgmap v340: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:13 smithi110 ceph-mon[49387]: pgmap v341: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:13 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:13 smithi053 ceph-mon[64292]: pgmap v341: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:13 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:15.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:15 smithi110 ceph-mon[49387]: pgmap v342: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:15.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:15 smithi053 ceph-mon[64292]: pgmap v342: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:17 smithi110 ceph-mon[49387]: pgmap v343: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:17 smithi053 ceph-mon[64292]: pgmap v343: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:19 smithi110 ceph-mon[49387]: pgmap v344: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:19.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:19 smithi053 ceph-mon[64292]: pgmap v344: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:21 smithi110 ceph-mon[49387]: pgmap v345: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:21.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:21 smithi053 ceph-mon[64292]: pgmap v345: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:22.772 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph orch ps' 2024-09-13T05:41:23.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:23 smithi053 ceph-mon[64292]: from='client.34550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:41:23.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:23 smithi053 ceph-mon[64292]: pgmap v346: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:23 smithi110 ceph-mon[49387]: from='client.34550 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:41:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:23 smithi110 ceph-mon[49387]: pgmap v346: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (10m) 24s ago 20m 20.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (7m) 24s ago 20m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (7m) 52s ago 20m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (30s) 24s ago 20m 38.7M - 10.4.0 c8b91775d855 c67be7242dd6 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (94s) 24s ago 14m 39.9M - 19.3.0-4911-g9d608eb1 0ce38589d192 c5ed825917f6 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (73s) 52s ago 14m 13.0M - 19.3.0-4911-g9d608eb1 0ce38589d192 03edcbdbc507 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (10m) 24s ago 22m 532M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:41:24.792 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (9m) 52s ago 20m 457M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (8m) 24s ago 22m 63.5M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (8m) 52s ago 20m 49.6M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (60s) 24s ago 14m 68.6M - 5.9 0ce38589d192 a660bae25d54 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (55s) 52s ago 14m 19.7M - 5.9 0ce38589d192 7e2784544425 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (10m) 24s ago 20m 10.9M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (10m) 52s ago 19m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (6m) 24s ago 19m 329M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (6m) 24s ago 18m 363M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (5m) 24s ago 18m 315M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (4m) 24s ago 17m 337M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (3m) 52s ago 17m 318M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (3m) 52s ago 17m 270M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 caa8bc291506 2024-09-13T05:41:24.793 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (2m) 52s ago 16m 240M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 260b18297eb5 2024-09-13T05:41:24.794 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (116s) 52s ago 16m 269M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 9afa18d580fe 2024-09-13T05:41:24.794 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (9m) 24s ago 19m 56.7M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:41:25.438 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph versions' 2024-09-13T05:41:25.744 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:25 smithi053 ceph-mon[64292]: pgmap v347: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:25.744 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:25 smithi053 ceph-mon[64292]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:41:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:25 smithi110 ceph-mon[49387]: pgmap v347: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:25 smithi110 ceph-mon[49387]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 8 2024-09-13T05:41:27.521 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:41:27.522 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:41:27.522 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:41:27.522 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:41:27.522 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:41:27.522 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 14 2024-09-13T05:41:27.522 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:41:27.522 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:41:27.832 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:27 smithi053 ceph-mon[64292]: pgmap v348: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:27.832 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:27 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/2428549645' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:27.832 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:27 smithi110 ceph-mon[49387]: pgmap v348: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:27 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/2428549645' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:41:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:28.140 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-13T05:41:29.642 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:29 smithi053 ceph-mon[64292]: pgmap v349: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:29.876 INFO:teuthology.orchestra.run.smithi053.stdout:wait for servicemap items w/ changing names to refresh 2024-09-13T05:41:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:29 smithi110 ceph-mon[49387]: pgmap v349: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:30.493 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'sleep 60' 2024-09-13T05:41:31.714 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:31 smithi053 ceph-mon[64292]: pgmap v350: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:31.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:31 smithi110 ceph-mon[49387]: pgmap v350: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:33 smithi110 ceph-mon[49387]: pgmap v351: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:33.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:33 smithi053 ceph-mon[64292]: pgmap v351: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:35 smithi110 ceph-mon[49387]: pgmap v352: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:35.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:35 smithi053 ceph-mon[64292]: pgmap v352: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:37 smithi110 ceph-mon[49387]: pgmap v353: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:37.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:37 smithi053 ceph-mon[64292]: pgmap v353: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:39.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:39 smithi110 ceph-mon[49387]: pgmap v354: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:39.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:39 smithi053 ceph-mon[64292]: pgmap v354: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:41.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:41 smithi110 ceph-mon[49387]: pgmap v355: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:41.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:41 smithi053 ceph-mon[64292]: pgmap v355: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:42.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:42.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:43.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:43 smithi110 ceph-mon[49387]: pgmap v356: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:43.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:43 smithi053 ceph-mon[64292]: pgmap v356: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:45.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:45 smithi110 ceph-mon[49387]: pgmap v357: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:45.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:45 smithi053 ceph-mon[64292]: pgmap v357: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:47.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:47 smithi110 ceph-mon[49387]: pgmap v358: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:47.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:47 smithi053 ceph-mon[64292]: pgmap v358: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:49.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:49 smithi110 ceph-mon[49387]: pgmap v359: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:49.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:49 smithi053 ceph-mon[64292]: pgmap v359: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:51.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:51 smithi110 ceph-mon[49387]: pgmap v360: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:51 smithi053 ceph-mon[64292]: pgmap v360: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:53.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:53 smithi110 ceph-mon[49387]: pgmap v361: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:53.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:53 smithi053 ceph-mon[64292]: pgmap v361: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:55.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:55 smithi110 ceph-mon[49387]: pgmap v362: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:55.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:55 smithi053 ceph-mon[64292]: pgmap v362: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:57 smithi110 ceph-mon[49387]: pgmap v363: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:57.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:57.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:57 smithi053 ceph-mon[64292]: pgmap v363: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:41:57.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:41:59.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:41:59 smithi110 ceph-mon[49387]: pgmap v364: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:41:59.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:41:59 smithi053 ceph-mon[64292]: pgmap v364: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:01.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:01 smithi110 ceph-mon[49387]: pgmap v365: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:01.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:01 smithi053 ceph-mon[64292]: pgmap v365: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:03.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:03 smithi110 ceph-mon[49387]: pgmap v366: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:03.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:03 smithi053 ceph-mon[64292]: pgmap v366: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:05.996 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:05 smithi110 ceph-mon[49387]: pgmap v367: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:05.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:42:05.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:05.997 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:05 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:05 smithi053 ceph-mon[64292]: pgmap v367: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T05:42:05.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:05.998 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:05 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:06 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:42:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:06 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:42:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:06 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:06.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:06 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:06.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:06 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T05:42:06.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:06 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T05:42:06.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:06 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:06.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:06 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' 2024-09-13T05:42:07.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:07 smithi110 ceph-mon[49387]: pgmap v368: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:07.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:07 smithi053 ceph-mon[64292]: pgmap v368: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:09.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:09 smithi110 ceph-mon[49387]: pgmap v369: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:09.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:09 smithi053 ceph-mon[64292]: pgmap v369: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:11.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:11 smithi110 ceph-mon[49387]: pgmap v370: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:11.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:11 smithi053 ceph-mon[64292]: pgmap v370: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:12.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:12 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:12.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:12 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:13.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:13 smithi110 ceph-mon[49387]: pgmap v371: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:13.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:13 smithi053 ceph-mon[64292]: pgmap v371: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:15.995 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:15 smithi110 ceph-mon[49387]: pgmap v372: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:15.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:15 smithi053 ceph-mon[64292]: pgmap v372: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:17.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:17 smithi110 ceph-mon[49387]: pgmap v373: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:17.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:17 smithi053 ceph-mon[64292]: pgmap v373: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:19.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:19 smithi110 ceph-mon[49387]: pgmap v374: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:19.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:19 smithi053 ceph-mon[64292]: pgmap v374: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:21.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:21 smithi110 ceph-mon[49387]: pgmap v375: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:21.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:21 smithi053 ceph-mon[64292]: pgmap v375: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:23.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:23 smithi110 ceph-mon[49387]: pgmap v376: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:23.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:23 smithi053 ceph-mon[64292]: pgmap v376: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:25.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:25 smithi110 ceph-mon[49387]: pgmap v377: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:25.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:25 smithi053 ceph-mon[64292]: pgmap v377: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:27 smithi110 ceph-mon[49387]: pgmap v378: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:27.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:27 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:27.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:27 smithi053 ceph-mon[64292]: pgmap v378: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:27.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:27 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:29.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:29 smithi110 ceph-mon[49387]: pgmap v379: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:29.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:29 smithi053 ceph-mon[64292]: pgmap v379: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:31.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:31 smithi110 ceph-mon[49387]: pgmap v380: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:31.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:31 smithi053 ceph-mon[64292]: pgmap v380: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:32.922 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph orch ps' 2024-09-13T05:42:33.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:33 smithi053 ceph-mon[64292]: pgmap v381: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:33.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:33 smithi110 ceph-mon[49387]: pgmap v381: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:34.926 INFO:teuthology.orchestra.run.smithi053.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:alertmanager.smithi053 smithi053 *:9093,9094 running (12m) 95s ago 22m 20.5M - 0.27.0 11f11916f8cd d1ca589118b1 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi053 smithi053 running (8m) 95s ago 21m 6899k - 19.3.0-4911-g9d608eb1 0ce38589d192 5b88799f7eda 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:crash.smithi110 smithi110 running (8m) 2m ago 21m 6891k - 19.3.0-4911-g9d608eb1 0ce38589d192 2de96bd3dcd9 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:grafana.smithi053 smithi053 *:3000 running (100s) 95s ago 21m 38.7M - 10.4.0 c8b91775d855 c67be7242dd6 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi053.rzvrng smithi053 running (2m) 95s ago 16m 39.9M - 19.3.0-4911-g9d608eb1 0ce38589d192 c5ed825917f6 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:mds.foofs.smithi110.akbygi smithi110 running (2m) 2m ago 15m 13.0M - 19.3.0-4911-g9d608eb1 0ce38589d192 03edcbdbc507 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi053.ygzjut smithi053 *:8443,9283,8765 running (11m) 95s ago 23m 532M - 19.3.0-4911-g9d608eb1 0ce38589d192 1ef23d02fd0f 2024-09-13T05:42:34.927 INFO:teuthology.orchestra.run.smithi053.stdout:mgr.smithi110.rivldx smithi110 *:8443,9283,8765 running (10m) 2m ago 21m 457M - 19.3.0-4911-g9d608eb1 0ce38589d192 16355c9da4eb 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi053 smithi053 running (10m) 95s ago 23m 63.5M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7ce01e261ca3 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:mon.smithi110 smithi110 running (9m) 2m ago 21m 49.6M 2048M 19.3.0-4911-g9d608eb1 0ce38589d192 7c6115f393f9 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.0.0.smithi053.zidqnx smithi053 *:2049 running (2m) 95s ago 15m 68.6M - 5.9 0ce38589d192 a660bae25d54 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:nfs.foo.1.0.smithi110.fvztpo smithi110 *:2049 running (2m) 2m ago 15m 19.7M - 5.9 0ce38589d192 7e2784544425 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi053 smithi053 *:9100 running (11m) 95s ago 21m 10.9M - 1.7.0 72c9c2088986 78f4fe05abf7 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:node-exporter.smithi110 smithi110 *:9100 running (11m) 2m ago 21m 11.9M - 1.7.0 72c9c2088986 2cca32829a69 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:osd.0 smithi053 running (8m) 95s ago 20m 329M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 a83746e4a39d 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:osd.1 smithi053 running (7m) 95s ago 20m 363M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 025330b4adae 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:osd.2 smithi053 running (6m) 95s ago 19m 315M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 685d12d14368 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:osd.3 smithi053 running (5m) 95s ago 19m 337M 1937M 19.3.0-4911-g9d608eb1 0ce38589d192 91b91cf43436 2024-09-13T05:42:34.928 INFO:teuthology.orchestra.run.smithi053.stdout:osd.4 smithi110 running (4m) 2m ago 18m 318M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 c09b6fed14c0 2024-09-13T05:42:34.929 INFO:teuthology.orchestra.run.smithi053.stdout:osd.5 smithi110 running (4m) 2m ago 18m 270M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 caa8bc291506 2024-09-13T05:42:34.929 INFO:teuthology.orchestra.run.smithi053.stdout:osd.6 smithi110 running (3m) 2m ago 17m 240M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 260b18297eb5 2024-09-13T05:42:34.929 INFO:teuthology.orchestra.run.smithi053.stdout:osd.7 smithi110 running (3m) 2m ago 17m 269M 2704M 19.3.0-4911-g9d608eb1 0ce38589d192 9afa18d580fe 2024-09-13T05:42:34.929 INFO:teuthology.orchestra.run.smithi053.stdout:prometheus.smithi053 smithi053 *:9095 running (10m) 95s ago 20m 56.7M - 2.51.0 1d3b7f56885b 03d9ab6ee6ec 2024-09-13T05:42:35.499 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph orch upgrade status' 2024-09-13T05:42:35.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:35 smithi110 ceph-mon[49387]: pgmap v382: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:35.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:35 smithi053 ceph-mon[64292]: pgmap v382: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:36.789 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:36 smithi053 ceph-mon[64292]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:36.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:36 smithi110 ceph-mon[49387]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:37.613 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:42:37.613 INFO:teuthology.orchestra.run.smithi053.stdout: "target_image": null, 2024-09-13T05:42:37.614 INFO:teuthology.orchestra.run.smithi053.stdout: "in_progress": false, 2024-09-13T05:42:37.614 INFO:teuthology.orchestra.run.smithi053.stdout: "which": "", 2024-09-13T05:42:37.614 INFO:teuthology.orchestra.run.smithi053.stdout: "services_complete": [], 2024-09-13T05:42:37.614 INFO:teuthology.orchestra.run.smithi053.stdout: "progress": null, 2024-09-13T05:42:37.614 INFO:teuthology.orchestra.run.smithi053.stdout: "message": "", 2024-09-13T05:42:37.614 INFO:teuthology.orchestra.run.smithi053.stdout: "is_paused": false 2024-09-13T05:42:37.614 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:42:37.973 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:37 smithi053 ceph-mon[64292]: pgmap v383: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:37.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:37 smithi110 ceph-mon[49387]: pgmap v383: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:38.243 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph health detail' 2024-09-13T05:42:38.880 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:38 smithi053 ceph-mon[64292]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:39.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:38 smithi110 ceph-mon[49387]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:39.870 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:39 smithi053 ceph-mon[64292]: pgmap v384: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:40.225 INFO:teuthology.orchestra.run.smithi053.stdout:HEALTH_OK 2024-09-13T05:42:40.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:39 smithi110 ceph-mon[49387]: pgmap v384: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:40.799 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:40 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3229072042' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:42:40.824 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph versions' 2024-09-13T05:42:41.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:40 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3229072042' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T05:42:42.131 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:41 smithi053 ceph-mon[64292]: pgmap v385: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:42.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:41 smithi110 ceph-mon[49387]: pgmap v385: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:42.947 INFO:teuthology.orchestra.run.smithi053.stdout:{ 2024-09-13T05:42:42.947 INFO:teuthology.orchestra.run.smithi053.stdout: "mon": { 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "mgr": { 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "osd": { 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 8 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "mds": { 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 2 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:42:42.948 INFO:teuthology.orchestra.run.smithi053.stdout: "overall": { 2024-09-13T05:42:42.949 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)": 14 2024-09-13T05:42:42.949 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:42:42.949 INFO:teuthology.orchestra.run.smithi053.stdout:} 2024-09-13T05:42:43.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:42 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:43.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:42 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:43.657 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-13T05:42:43.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:43 smithi053 ceph-mon[64292]: pgmap v386: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:43.972 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:43 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3590285172' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:42:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:43 smithi110 ceph-mon[49387]: pgmap v386: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:44.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:43 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3590285172' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:42:45.639 INFO:teuthology.orchestra.run.smithi053.stdout:true 2024-09-13T05:42:45.977 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:45 smithi053 ceph-mon[64292]: pgmap v387: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:45.977 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:45 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3536696664' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:42:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:45 smithi110 ceph-mon[49387]: pgmap v387: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:46.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:45 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3536696664' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:42:46.285 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -e sha1=9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-13T05:42:47.815 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:47 smithi053 ceph-mon[64292]: pgmap v388: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:48.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:47 smithi110 ceph-mon[49387]: pgmap v388: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:48.391 INFO:teuthology.orchestra.run.smithi053.stdout: "ceph version 19.3.0-4911-g9d608eb1 (9d608eb13dd49ee7b4b1637dbcf4f78b31ddd3d3) squid (dev)" 2024-09-13T05:42:48.968 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:48 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/3885995056' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:42:48.996 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-13T05:42:48.997 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-13T05:42:48.998 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T05:42:49.145 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:48 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/3885995056' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T05:42:49.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:49 smithi053 ceph-mon[64292]: pgmap v389: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:50.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:49 smithi110 ceph-mon[49387]: pgmap v389: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:42:50.984 INFO:teuthology.orchestra.run.smithi053.stdout: 2024-09-13T05:42:50.985 INFO:teuthology.orchestra.run.smithi053.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T05:19:56.091160Z", "last_refresh": "2024-09-13T05:40:59.887266Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T05:19:49.578973Z", "last_refresh": "2024-09-13T05:40:32.618575Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-13T05:19:52.700899Z", "last_refresh": "2024-09-13T05:40:59.887408Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T05:26:28.825146Z", "last_refresh": "2024-09-13T05:40:32.619177Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T05:19:47.487653Z", "last_refresh": "2024-09-13T05:40:32.618671Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi053:172.21.15.53=smithi053", "smithi110:172.21.15.110=smithi110"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T05:20:40.946189Z", "last_refresh": "2024-09-13T05:40:32.618757Z", "running": 2, "size": 2}}, {"events": ["2024-09-13T05:42:05.689090Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-13T05:26:49.607488Z", "last_refresh": "2024-09-13T05:40:32.619247Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T05:19:54.826039Z", "last_refresh": "2024-09-13T05:40:32.618832Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff", "last_refresh": "2024-09-13T05:40:32.618902Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T05:19:51.449591Z", "last_refresh": "2024-09-13T05:40:59.887552Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T05:42:51.598 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-13T05:42:51.599 DEBUG:teuthology.parallel:result is None 2024-09-13T05:42:51.599 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T05:42:51.611 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:42:51.612 DEBUG:teuthology.orchestra.run.smithi053:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-13T05:42:51.638 INFO:teuthology.orchestra.run.smithi053.stderr:+ umount /mnt/foo 2024-09-13T05:42:51.667 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T05:42:51.678 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:42:51.679 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-13T05:42:51.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:51 smithi053 ceph-mon[64292]: pgmap v390: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:52.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:51 smithi110 ceph-mon[49387]: pgmap v390: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:52.966 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:52 smithi053 ceph-mon[64292]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:42:53.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:52 smithi110 ceph-mon[49387]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T05:42:53.717 INFO:teuthology.orchestra.run.smithi053.stdout: "foo" 2024-09-13T05:42:53.997 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:53 smithi053 ceph-mon[64292]: pgmap v391: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:54.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:53 smithi110 ceph-mon[49387]: pgmap v391: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:54.516 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-13T05:42:55.070 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:54 smithi053 ceph-mon[64292]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:55.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:54 smithi110 ceph-mon[49387]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:56.066 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:55 smithi053 ceph-mon[64292]: pgmap v392: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:56.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:55 smithi110 ceph-mon[49387]: pgmap v392: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout:[ 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: { 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: "access_type": "RW", 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: "clients": [], 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: "cluster_id": "foo", 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: "export_id": 1, 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: "fsal": { 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: "fs_name": "foofs", 2024-09-13T05:42:56.593 INFO:teuthology.orchestra.run.smithi053.stdout: "name": "CEPH", 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "user_id": "nfs.foo.1" 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: }, 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "path": "/", 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "protocols": [ 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: 4 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: ], 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "pseudo": "/fake", 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "security_label": true, 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "squash": "none", 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "transports": [ 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: "TCP" 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: ] 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout: } 2024-09-13T05:42:56.594 INFO:teuthology.orchestra.run.smithi053.stdout:] 2024-09-13T05:42:57.225 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-13T05:42:58.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:57 smithi110 ceph-mon[49387]: pgmap v393: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:58.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:57 smithi110 ceph-mon[49387]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:58.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:57 smithi110 ceph-mon[49387]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:57 smithi053 ceph-mon[64292]: pgmap v393: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:42:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:57 smithi053 ceph-mon[64292]: from='client.34594 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T05:42:58.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:57 smithi053 ceph-mon[64292]: from='mgr.34104 172.21.15.53:0/51621167' entity='mgr.smithi053.ygzjut' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T05:42:59.128 INFO:teuthology.orchestra.run.smithi053.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-13T05:42:59.128 INFO:teuthology.orchestra.run.smithi053.stdout:foo grace 2024-09-13T05:42:59.128 INFO:teuthology.orchestra.run.smithi053.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-13T05:42:59.128 INFO:teuthology.orchestra.run.smithi053.stdout:foo conf-nfs.foo 2024-09-13T05:42:59.128 INFO:teuthology.orchestra.run.smithi053.stdout:foo export-1 2024-09-13T05:42:59.778 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T05:42:59.788 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:42:59.788 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-13T05:42:59.788 DEBUG:teuthology.orchestra.run.smithi053:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-13T05:42:59.788 DEBUG:teuthology.orchestra.run.smithi053:> ' 2024-09-13T05:43:00.099 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:59 smithi053 ceph-mon[64292]: pgmap v394: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:43:00.099 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:42:59 smithi053 ceph-mon[64292]: mgrmap e44: smithi053.ygzjut(active, since 10m), standbys: smithi110.rivldx 2024-09-13T05:43:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:59 smithi110 ceph-mon[49387]: pgmap v394: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T05:43:00.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:42:59 smithi110 ceph-mon[49387]: mgrmap e44: smithi053.ygzjut(active, since 10m), standbys: smithi110.rivldx 2024-09-13T05:43:01.534 INFO:teuthology.orchestra.run.smithi053.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-13T05:43:01.777 INFO:teuthology.orchestra.run.smithi053.stderr:+ [[ 8 -gt 2 ]] 2024-09-13T05:43:02.064 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:01 smithi053 ceph-mon[64292]: pgmap v395: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:43:02.065 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:01 smithi053 ceph-mon[64292]: from='client.? 172.21.15.53:0/1413750243' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-13T05:43:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:01 smithi110 ceph-mon[49387]: pgmap v395: 97 pgs: 97 active+clean; 234 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T05:43:02.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:01 smithi110 ceph-mon[49387]: from='client.? 172.21.15.53:0/1413750243' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-13T05:43:02.381 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T05:43:02.392 INFO:tasks.cephadm:Teardown begin 2024-09-13T05:43:02.393 DEBUG:teuthology.orchestra.run.smithi053:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T05:43:02.423 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T05:43:02.456 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T05:43:02.457 DEBUG:teuthology.orchestra.run.smithi053:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T05:43:02.481 DEBUG:teuthology.orchestra.run.smithi110:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T05:43:02.515 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T05:43:02.515 INFO:tasks.cephadm.mon.smithi053:Stopping mon.smithi053... 2024-09-13T05:43:02.516 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053 2024-09-13T05:43:02.747 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:02 smithi053 systemd[1]: Stopping Ceph mon.smithi053 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[64260]: 2024-09-13T05:43:02.833+0000 7fe09c40d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi053 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:02 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053[64260]: 2024-09-13T05:43:02.833+0000 7fe09c40d640 -1 mon.smithi053@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T05:43:03.247 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:02 smithi053 podman[94233]: 2024-09-13 05:43:02.967537873 +0000 UTC m=+0.251894159 container died 7ce01e261ca36ff49f719c8ab239a4b8cc930ea8fac1a903925cec914d05db36 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:43:03.627 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:03 smithi053 podman[94233]: 2024-09-13 05:43:03.351192229 +0000 UTC m=+0.635548515 container cleanup 7ce01e261ca36ff49f719c8ab239a4b8cc930ea8fac1a903925cec914d05db36 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:43:03.628 INFO:journalctl@ceph.mon.smithi053.smithi053.stdout:Sep 13 05:43:03 smithi053 bash[94233]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi053 2024-09-13T05:43:03.765 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi053.service' 2024-09-13T05:43:03.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:03.834 INFO:tasks.cephadm.mon.smithi053:Stopped mon.smithi053 2024-09-13T05:43:03.834 INFO:tasks.cephadm.mon.smithi110:Stopping mon.smithi110... 2024-09-13T05:43:03.834 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi110 2024-09-13T05:43:04.244 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:03 smithi110 systemd[1]: Stopping Ceph mon.smithi110 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:04.614 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:04 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110[49383]: 2024-09-13T05:43:04.295+0000 7f8277fb9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi110 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:04.614 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:04 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110[49383]: 2024-09-13T05:43:04.295+0000 7f8277fb9640 -1 mon.smithi110@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T05:43:04.614 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:04 smithi110 podman[68456]: 2024-09-13 05:43:04.327181793 +0000 UTC m=+0.296316749 container died 7c6115f393f971d7a7082dc8e15e6dbb19bf1583a3a06930882431c2e32f0321 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:43:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:04 smithi110 podman[68456]: 2024-09-13 05:43:04.788615367 +0000 UTC m=+0.757750365 container cleanup 7c6115f393f971d7a7082dc8e15e6dbb19bf1583a3a06930882431c2e32f0321 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T05:43:04.994 INFO:journalctl@ceph.mon.smithi110.smithi110.stdout:Sep 13 05:43:04 smithi110 bash[68456]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-mon-smithi110 2024-09-13T05:43:05.231 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@mon.smithi110.service' 2024-09-13T05:43:05.274 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:05.274 INFO:tasks.cephadm.mon.smithi110:Stopped mon.smithi110 2024-09-13T05:43:05.275 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T05:43:05.275 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.0 2024-09-13T05:43:05.598 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:05 smithi053 systemd[1]: Stopping Ceph osd.0 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:05.997 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:05 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[69205]: 2024-09-13T05:43:05.597+0000 7ff4048b2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:05.997 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:05 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[69205]: 2024-09-13T05:43:05.597+0000 7ff4048b2640 -1 osd.0 155 *** Got signal Terminated *** 2024-09-13T05:43:05.997 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:05 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0[69205]: 2024-09-13T05:43:05.597+0000 7ff4048b2640 -1 osd.0 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:43:11.324 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:11 smithi053 podman[94365]: 2024-09-13 05:43:11.065396423 +0000 UTC m=+5.591840290 container died a83746e4a39d264962a69b005c43599c1ce5552fa6b1a6e554254a8827aa11ce (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T05:43:11.616 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:11 smithi053 podman[94365]: 2024-09-13 05:43:11.423166846 +0000 UTC m=+5.949610767 container cleanup a83746e4a39d264962a69b005c43599c1ce5552fa6b1a6e554254a8827aa11ce (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:11.616 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:11 smithi053 bash[94365]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0 2024-09-13T05:43:11.997 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:11 smithi053 podman[94381]: 2024-09-13 05:43:11.61655741 +0000 UTC m=+0.546158482 container remove a83746e4a39d264962a69b005c43599c1ce5552fa6b1a6e554254a8827aa11ce (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, GIT_BRANCH=HEAD) 2024-09-13T05:43:12.312 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:12 smithi053 podman[94470]: 2024-09-13 05:43:12.059219987 +0000 UTC m=+0.113633905 container create a04ba8f825fbc9354cd28a06d33c7bf626948d2dd2045666ff7a2d86cccef059 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:12.312 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:12 smithi053 podman[94470]: 2024-09-13 05:43:11.962278659 +0000 UTC m=+0.016692587 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:43:12.312 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:12 smithi053 podman[94470]: 2024-09-13 05:43:12.312158455 +0000 UTC m=+0.366572376 container init a04ba8f825fbc9354cd28a06d33c7bf626948d2dd2045666ff7a2d86cccef059 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-13T05:43:12.747 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:12 smithi053 podman[94470]: 2024-09-13 05:43:12.315683695 +0000 UTC m=+0.370097616 container start a04ba8f825fbc9354cd28a06d33c7bf626948d2dd2045666ff7a2d86cccef059 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:43:12.747 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:12 smithi053 podman[94470]: 2024-09-13 05:43:12.412033546 +0000 UTC m=+0.466447482 container attach a04ba8f825fbc9354cd28a06d33c7bf626948d2dd2045666ff7a2d86cccef059 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T05:43:12.747 INFO:journalctl@ceph.osd.0.smithi053.stdout:Sep 13 05:43:12 smithi053 podman[94470]: 2024-09-13 05:43:12.523110493 +0000 UTC m=+0.577524414 container died a04ba8f825fbc9354cd28a06d33c7bf626948d2dd2045666ff7a2d86cccef059 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-13T05:43:13.168 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T05:43:13.214 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:13.214 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T05:43:13.214 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T05:43:13.214 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.1 2024-09-13T05:43:13.497 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:13 smithi053 systemd[1]: Stopping Ceph osd.1 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:13.997 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[71627]: 2024-09-13T05:43:13.585+0000 7f01426a5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:13.997 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[71627]: 2024-09-13T05:43:13.585+0000 7f01426a5640 -1 osd.1 155 *** Got signal Terminated *** 2024-09-13T05:43:13.997 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:13 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1[71627]: 2024-09-13T05:43:13.585+0000 7f01426a5640 -1 osd.1 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:43:19.331 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:19 smithi053 podman[94571]: 2024-09-13 05:43:19.067063532 +0000 UTC m=+5.620700472 container died 025330b4adae5be123710d51080a6fecdc560f5a35bf336f71044fa2233b89d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-13T05:43:19.596 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:19 smithi053 podman[94571]: 2024-09-13 05:43:19.403028822 +0000 UTC m=+5.956665764 container cleanup 025330b4adae5be123710d51080a6fecdc560f5a35bf336f71044fa2233b89d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:43:19.596 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:19 smithi053 bash[94571]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1 2024-09-13T05:43:19.997 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:19 smithi053 podman[94602]: 2024-09-13 05:43:19.595849035 +0000 UTC m=+0.526730272 container remove 025330b4adae5be123710d51080a6fecdc560f5a35bf336f71044fa2233b89d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:43:20.282 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:20 smithi053 podman[94691]: 2024-09-13 05:43:19.935233125 +0000 UTC m=+0.016940798 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:43:20.282 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:20 smithi053 podman[94691]: 2024-09-13 05:43:20.038237038 +0000 UTC m=+0.119944703 container create b84e4bd022301a359e405c5ca1c6c9d90204363fb96ecc8b396f31e06231bd48 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:20.283 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:20 smithi053 podman[94691]: 2024-09-13 05:43:20.282584277 +0000 UTC m=+0.364291940 container init b84e4bd022301a359e405c5ca1c6c9d90204363fb96ecc8b396f31e06231bd48 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-13T05:43:20.747 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:20 smithi053 podman[94691]: 2024-09-13 05:43:20.286039953 +0000 UTC m=+0.367747623 container start b84e4bd022301a359e405c5ca1c6c9d90204363fb96ecc8b396f31e06231bd48 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T05:43:20.747 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:20 smithi053 podman[94691]: 2024-09-13 05:43:20.37453563 +0000 UTC m=+0.456243305 container attach b84e4bd022301a359e405c5ca1c6c9d90204363fb96ecc8b396f31e06231bd48 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2) 2024-09-13T05:43:20.748 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:20 smithi053 podman[94691]: 2024-09-13 05:43:20.487370811 +0000 UTC m=+0.569078475 container died b84e4bd022301a359e405c5ca1c6c9d90204363fb96ecc8b396f31e06231bd48 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T05:43:21.036 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T05:43:21.063 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:21 smithi053 podman[94714]: 2024-09-13 05:43:21.011552628 +0000 UTC m=+0.516749282 container remove b84e4bd022301a359e405c5ca1c6c9d90204363fb96ecc8b396f31e06231bd48 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T05:43:21.063 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:21 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T05:43:21.063 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:21 smithi053 systemd[1]: Stopped Ceph osd.1 for 9f95c28e-718f-11ef-bceb-c7b262605968. 2024-09-13T05:43:21.063 INFO:journalctl@ceph.osd.1.smithi053.stdout:Sep 13 05:43:21 smithi053 systemd[1]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.1.service: Consumed 1min 25.762s CPU time. 2024-09-13T05:43:21.081 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:21.081 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T05:43:21.081 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T05:43:21.081 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.2 2024-09-13T05:43:21.440 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:21 smithi053 systemd[1]: Stopping Ceph osd.2 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:21.747 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:21 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[74090]: 2024-09-13T05:43:21.439+0000 7f23f8ca5640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:21.747 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:21 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[74090]: 2024-09-13T05:43:21.439+0000 7f23f8ca5640 -1 osd.2 155 *** Got signal Terminated *** 2024-09-13T05:43:21.747 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:21 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2[74090]: 2024-09-13T05:43:21.439+0000 7f23f8ca5640 -1 osd.2 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:43:27.216 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:26 smithi053 podman[94792]: 2024-09-13 05:43:26.938124859 +0000 UTC m=+5.615673172 container died 685d12d14368132f076f30e23a5295d0a95182d04a641def87155d0840146cd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:27.491 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:27 smithi053 podman[94792]: 2024-09-13 05:43:27.281249171 +0000 UTC m=+5.958797474 container cleanup 685d12d14368132f076f30e23a5295d0a95182d04a641def87155d0840146cd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-13T05:43:27.491 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:27 smithi053 bash[94792]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2 2024-09-13T05:43:27.747 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:27 smithi053 podman[94809]: 2024-09-13 05:43:27.491490057 +0000 UTC m=+0.552268934 container remove 685d12d14368132f076f30e23a5295d0a95182d04a641def87155d0840146cd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T05:43:28.203 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:27 smithi053 podman[94897]: 2024-09-13 05:43:27.812071809 +0000 UTC m=+0.016958992 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:43:28.203 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:27 smithi053 podman[94897]: 2024-09-13 05:43:27.933968018 +0000 UTC m=+0.138855193 container create fcf9fdd9e384626e7364fb51ff3ce367fea00f1e3ab8b275f3149afb63564ac8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T05:43:28.498 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:28 smithi053 podman[94897]: 2024-09-13 05:43:28.203227434 +0000 UTC m=+0.408114616 container init fcf9fdd9e384626e7364fb51ff3ce367fea00f1e3ab8b275f3149afb63564ac8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:28.498 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:28 smithi053 podman[94897]: 2024-09-13 05:43:28.207381502 +0000 UTC m=+0.412268679 container start fcf9fdd9e384626e7364fb51ff3ce367fea00f1e3ab8b275f3149afb63564ac8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:28.498 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:28 smithi053 podman[94897]: 2024-09-13 05:43:28.311657937 +0000 UTC m=+0.516545132 container attach fcf9fdd9e384626e7364fb51ff3ce367fea00f1e3ab8b275f3149afb63564ac8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731) 2024-09-13T05:43:28.498 INFO:journalctl@ceph.osd.2.smithi053.stdout:Sep 13 05:43:28 smithi053 podman[94897]: 2024-09-13 05:43:28.402376676 +0000 UTC m=+0.607263849 container died fcf9fdd9e384626e7364fb51ff3ce367fea00f1e3ab8b275f3149afb63564ac8 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True) 2024-09-13T05:43:29.004 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T05:43:29.047 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:29.048 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T05:43:29.048 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T05:43:29.048 DEBUG:teuthology.orchestra.run.smithi053:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.3 2024-09-13T05:43:29.247 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:29 smithi053 systemd[1]: Stopping Ceph osd.3 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:29.747 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:29 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[76488]: 2024-09-13T05:43:29.368+0000 7f0498270640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:29.747 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:29 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[76488]: 2024-09-13T05:43:29.368+0000 7f0498270640 -1 osd.3 155 *** Got signal Terminated *** 2024-09-13T05:43:29.747 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:29 smithi053 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3[76488]: 2024-09-13T05:43:29.368+0000 7f0498270640 -1 osd.3 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:43:31.494 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:31 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:31.196+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:32.477 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:32 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:32.190+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:32.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:32 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:32.477+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:33.464 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:33 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:33.159+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:33.740 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:33 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:33.464+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:33.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:33 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:33.740+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:34.420 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:34 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:34.112+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:34.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:34 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:34.420+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:34.997 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:34 smithi053 podman[95000]: 2024-09-13 05:43:34.726847554 +0000 UTC m=+5.448857067 container died 91b91cf43436c484136f60e9693e963a126a7b74cf7c2b26722f2a86a51c5400 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T05:43:35.064 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:34 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:34.754+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:35.361 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 podman[95000]: 2024-09-13 05:43:35.093310674 +0000 UTC m=+5.815320187 container cleanup 91b91cf43436c484136f60e9693e963a126a7b74cf7c2b26722f2a86a51c5400 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.license=GPLv2) 2024-09-13T05:43:35.361 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 bash[95000]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3 2024-09-13T05:43:35.433 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:35 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:35.064+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:35.693 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 podman[95036]: 2024-09-13 05:43:35.361540495 +0000 UTC m=+0.629391625 container remove 91b91cf43436c484136f60e9693e963a126a7b74cf7c2b26722f2a86a51c5400 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:43:35.711 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:35 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:35.433+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:35.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:35 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:35.711+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:35.997 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 podman[95126]: 2024-09-13 05:43:35.596277219 +0000 UTC m=+0.016571420 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:43:35.997 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 podman[95126]: 2024-09-13 05:43:35.695661407 +0000 UTC m=+0.115955596 container create e1df5ff54542491b3b522d27e5b00ec6f3e36b8829a8c4987eec5a59d9e3851a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T05:43:35.997 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 podman[95126]: 2024-09-13 05:43:35.923566649 +0000 UTC m=+0.343860837 container init e1df5ff54542491b3b522d27e5b00ec6f3e36b8829a8c4987eec5a59d9e3851a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-13T05:43:35.997 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 podman[95126]: 2024-09-13 05:43:35.926922126 +0000 UTC m=+0.347216323 container start e1df5ff54542491b3b522d27e5b00ec6f3e36b8829a8c4987eec5a59d9e3851a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T05:43:36.410 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:35 smithi053 podman[95126]: 2024-09-13 05:43:35.998332734 +0000 UTC m=+0.418626924 container attach e1df5ff54542491b3b522d27e5b00ec6f3e36b8829a8c4987eec5a59d9e3851a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T05:43:36.410 INFO:journalctl@ceph.osd.3.smithi053.stdout:Sep 13 05:43:36 smithi053 podman[95126]: 2024-09-13 05:43:36.119418678 +0000 UTC m=+0.539712867 container died e1df5ff54542491b3b522d27e5b00ec6f3e36b8829a8c4987eec5a59d9e3851a (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:43:36.417 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:36.101+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:36.417 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:36.325+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:36.684 DEBUG:teuthology.orchestra.run.smithi053:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T05:43:36.699 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:36.416+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:36.727 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:36.728 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T05:43:36.728 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T05:43:36.728 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.4 2024-09-13T05:43:36.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:36 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:36.699+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:36.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:36 smithi110 systemd[1]: Stopping Ceph osd.4 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:37.359 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:37.070+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:37.360 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:37.200+0000 7f0704cfb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:37.360 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:37.200+0000 7f0704cfb640 -1 osd.4 155 *** Got signal Terminated *** 2024-09-13T05:43:37.360 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:37.200+0000 7f0704cfb640 -1 osd.4 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:43:37.738 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:37.359+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:37.738 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:37.411+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:37.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:37 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:37.737+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:38.366 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:38 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:38.056+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:38.740 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:38 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:38.397+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:38.741 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:38 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:38.366+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:38.994 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:38 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:38.740+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:39.331 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:39 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:39.037+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:39.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:39 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:39.331+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:39.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:39 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:39.415+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:40.035 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:39 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:39.770+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:40.312 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:40 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:40.035+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:40.313 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:40 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:40.035+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:40.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:40 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:40.312+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:40.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:40 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:40.395+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:40.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:40 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:40.395+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:41.050 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:40 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:40.786+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:41.400 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:41.272+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:41.401 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:41.050+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:41.401 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:41.050+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:41.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:41.400+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:41.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:41.400+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:42.076 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:41.769+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.345786+0000 front 2024-09-13T05:43:10.345719+0000 (oldest deadline 2024-09-13T05:43:33.245618+0000) 2024-09-13T05:43:42.076 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:41 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4[53284]: 2024-09-13T05:43:41.769+0000 7f0701310640 -1 osd.4 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.148083+0000 front 2024-09-13T05:43:17.148017+0000 (oldest deadline 2024-09-13T05:43:41.247968+0000) 2024-09-13T05:43:42.406 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:42 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:42.261+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:42.406 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:42 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:42.076+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:42.406 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:42 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:42.076+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:42.707 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:42 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:42.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:42.707 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:42 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:42.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:42.993 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:42 smithi110 podman[68608]: 2024-09-13 05:43:42.70662503 +0000 UTC m=+5.776328488 container died c09b6fed14c0329944856266ed78967b07fccfd1271962721728023e75276787 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T05:43:43.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:43 smithi110 podman[68608]: 2024-09-13 05:43:43.192627748 +0000 UTC m=+6.262331209 container cleanup c09b6fed14c0329944856266ed78967b07fccfd1271962721728023e75276787 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-13T05:43:43.244 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:43 smithi110 bash[68608]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4 2024-09-13T05:43:43.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:43.071+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:43.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:43.071+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:43.744 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:43 smithi110 podman[68624]: 2024-09-13 05:43:43.444413945 +0000 UTC m=+0.732269584 container remove c09b6fed14c0329944856266ed78967b07fccfd1271962721728023e75276787 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:43:43.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:43.385+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:43.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:43.385+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:43.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:43.254+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:43.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:43 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:43.254+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:44.022 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:43 smithi110 podman[68711]: 2024-09-13 05:43:43.664038261 +0000 UTC m=+0.015989794 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:43:44.022 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:43 smithi110 podman[68711]: 2024-09-13 05:43:43.77428962 +0000 UTC m=+0.126241195 container create 34198aefe3402051a154a239ef0b2d64d3bd6e2f4f132a5720505d0c6c4b22ee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:43:44.022 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:44 smithi110 podman[68711]: 2024-09-13 05:43:44.001121998 +0000 UTC m=+0.353073530 container init 34198aefe3402051a154a239ef0b2d64d3bd6e2f4f132a5720505d0c6c4b22ee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=) 2024-09-13T05:43:44.022 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:44 smithi110 podman[68711]: 2024-09-13 05:43:44.004428746 +0000 UTC m=+0.356380279 container start 34198aefe3402051a154a239ef0b2d64d3bd6e2f4f132a5720505d0c6c4b22ee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T05:43:44.391 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:44 smithi110 podman[68711]: 2024-09-13 05:43:44.110025087 +0000 UTC m=+0.461976632 container attach 34198aefe3402051a154a239ef0b2d64d3bd6e2f4f132a5720505d0c6c4b22ee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T05:43:44.391 INFO:journalctl@ceph.osd.4.smithi110.stdout:Sep 13 05:43:44 smithi110 podman[68711]: 2024-09-13 05:43:44.197270961 +0000 UTC m=+0.549222499 container died 34198aefe3402051a154a239ef0b2d64d3bd6e2f4f132a5720505d0c6c4b22ee (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-13T05:43:44.392 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:44.021+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:44.392 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:44.021+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:44.392 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:44.267+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:44.392 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:44.267+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:44.730 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:44.391+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:44.730 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:44 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:44.391+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:44.756 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T05:43:44.794 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:44.795 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T05:43:44.795 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T05:43:44.795 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.5 2024-09-13T05:43:44.994 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:44 smithi110 systemd[1]: Stopping Ceph osd.5 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:45.421 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:45.257+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:45.421 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:45.257+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:45.421 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:45.070+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:45.422 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:45.070+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:45.422 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:45.175+0000 7fde479bb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:45.422 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:45.175+0000 7fde479bb640 -1 osd.5 155 *** Got signal Terminated *** 2024-09-13T05:43:45.422 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:45.175+0000 7fde479bb640 -1 osd.5 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:43:45.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:45.421+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:45.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:45 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:45.421+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:46.464 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:46 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:46.247+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:46.464 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:46 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:46.247+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:46.464 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:46 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:46.101+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:46.464 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:46 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:46.101+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:46.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:46 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:46.464+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:46.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:46 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:46.464+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:47.449 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:47 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:47.219+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:47.449 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:47 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:47.219+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:47.449 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:47 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:47.107+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:47.449 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:47 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:47.107+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:47.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:47 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:47.449+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:47.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:47 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:47.449+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:48.443 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:48.174+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:48.443 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:48.174+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:48.444 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:48.080+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:48.444 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:48.080+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:48.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:48.442+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:48.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:48 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:48.442+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:49.412 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:49.169+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:49.413 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:49.169+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:49.413 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:49.034+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:49.413 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:49.034+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:49.413 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:49.034+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:22.734853+0000 front 2024-09-13T05:43:22.734809+0000 (oldest deadline 2024-09-13T05:43:48.634788+0000) 2024-09-13T05:43:49.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:49.411+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:49.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:49.411+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:49.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:49.411+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:50.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:50 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:50.150+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:50.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:50 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:50.150+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:50.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:49.988+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.032113+0000 front 2024-09-13T05:43:10.032290+0000 (oldest deadline 2024-09-13T05:43:31.132213+0000) 2024-09-13T05:43:50.244 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:49.988+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:13.433238+0000 front 2024-09-13T05:43:13.433160+0000 (oldest deadline 2024-09-13T05:43:39.833692+0000) 2024-09-13T05:43:50.245 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:49 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5[55329]: 2024-09-13T05:43:49.988+0000 7fde3ffc8640 -1 osd.5 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:22.734853+0000 front 2024-09-13T05:43:22.734809+0000 (oldest deadline 2024-09-13T05:43:48.634788+0000) 2024-09-13T05:43:50.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:50 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:50.449+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:50.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:50 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:50.449+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:50.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:50 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:50.449+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:51.027 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:50 smithi110 podman[68825]: 2024-09-13 05:43:50.768864206 +0000 UTC m=+5.717737149 container died caa8bc291506e2a6499efc73cca5fc6d86db7ebd44570ff5571af8305b8dea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:51.369 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:51 smithi110 podman[68825]: 2024-09-13 05:43:51.19237101 +0000 UTC m=+6.141244011 container cleanup caa8bc291506e2a6499efc73cca5fc6d86db7ebd44570ff5571af8305b8dea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T05:43:51.369 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:51 smithi110 bash[68825]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5 2024-09-13T05:43:51.369 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:51.161+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:51.369 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:51.161+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:51.369 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:51.161+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:51.744 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:51 smithi110 podman[68841]: 2024-09-13 05:43:51.368778331 +0000 UTC m=+0.602089798 container remove caa8bc291506e2a6499efc73cca5fc6d86db7ebd44570ff5571af8305b8dea16 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T05:43:51.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:51.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:51.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:51.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:51.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:51 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:51.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:52.089 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:51 smithi110 podman[68930]: 2024-09-13 05:43:51.705898994 +0000 UTC m=+0.015803184 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:43:52.089 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:51 smithi110 podman[68930]: 2024-09-13 05:43:51.811531516 +0000 UTC m=+0.121435699 container create e66f8d747294c65ecec7c3974fb606dd21f8d5ab685e4e463308d0ccf0a6ff6c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:43:52.089 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:52 smithi110 podman[68930]: 2024-09-13 05:43:52.014074815 +0000 UTC m=+0.323978993 container init e66f8d747294c65ecec7c3974fb606dd21f8d5ab685e4e463308d0ccf0a6ff6c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:52.089 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:52 smithi110 podman[68930]: 2024-09-13 05:43:52.017283576 +0000 UTC m=+0.327187754 container start e66f8d747294c65ecec7c3974fb606dd21f8d5ab685e4e463308d0ccf0a6ff6c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-13T05:43:52.089 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:52 smithi110 podman[68930]: 2024-09-13 05:43:52.08904563 +0000 UTC m=+0.398949809 container attach e66f8d747294c65ecec7c3974fb606dd21f8d5ab685e4e463308d0ccf0a6ff6c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:43:52.407 INFO:journalctl@ceph.osd.5.smithi110.stdout:Sep 13 05:43:52 smithi110 podman[68930]: 2024-09-13 05:43:52.204500413 +0000 UTC m=+0.514404600 container died e66f8d747294c65ecec7c3974fb606dd21f8d5ab685e4e463308d0ccf0a6ff6c (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T05:43:52.407 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:52 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:52.200+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:52.407 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:52 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:52.200+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:52.407 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:52 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:52.200+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:52.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:52 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:52.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:52.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:52 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:52.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:52.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:52 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:52.405+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:52.771 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T05:43:52.810 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:43:52.810 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T05:43:52.811 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T05:43:52.811 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.6 2024-09-13T05:43:53.138 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:52 smithi110 systemd[1]: Stopping Ceph osd.6 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:43:53.392 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:53.245+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:53.392 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:53.245+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:53.392 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:53.245+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:53.392 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:53.137+0000 7f7a8459c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:43:53.393 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:53.137+0000 7f7a8459c640 -1 osd.6 155 *** Got signal Terminated *** 2024-09-13T05:43:53.393 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:53.137+0000 7f7a8459c640 -1 osd.6 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:43:53.393 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:53.391+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:53.393 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:53.391+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:53.393 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:53 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:53.391+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:54.719 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:54 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:54.384+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:54.720 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:54 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:54.384+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:54.720 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:54 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:54.384+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:54.720 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:54 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:54.267+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:54.720 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:54 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:54.267+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:54.720 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:54 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:54.267+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:55.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:55 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:55.428+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:55.745 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:55 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:55.428+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:55.745 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:55 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:55.428+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:55.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:55 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:55.251+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:55.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:55 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:55.251+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:55.745 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:55 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:55.251+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:56.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:56.248+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:56.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:56.248+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:56.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:56.248+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:56.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:56.408+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:56.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:56.408+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:56.745 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:56.408+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:56.745 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:56 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:56.408+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:33.484800+0000 front 2024-09-13T05:43:33.484786+0000 (oldest deadline 2024-09-13T05:43:55.784706+0000) 2024-09-13T05:43:57.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:57.381+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:10.082569+0000 front 2024-09-13T05:43:10.082503+0000 (oldest deadline 2024-09-13T05:43:35.982411+0000) 2024-09-13T05:43:57.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:57.381+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:15.982935+0000 front 2024-09-13T05:43:15.982953+0000 (oldest deadline 2024-09-13T05:43:40.082795+0000) 2024-09-13T05:43:57.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:57.381+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.983545+0000 front 2024-09-13T05:43:25.983665+0000 (oldest deadline 2024-09-13T05:43:48.883509+0000) 2024-09-13T05:43:57.494 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6[57366]: 2024-09-13T05:43:57.381+0000 7f7a7cba9640 -1 osd.6 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:33.484800+0000 front 2024-09-13T05:43:33.484786+0000 (oldest deadline 2024-09-13T05:43:55.784706+0000) 2024-09-13T05:43:57.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:57.208+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:57.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:57.208+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:57.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:57.208+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:57.495 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:57 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:57.208+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:43:58.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:58 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:58.205+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:58.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:58 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:58.205+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:58.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:58 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:58.205+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:58.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:58 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:58.205+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:43:58.981 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:58 smithi110 podman[69032]: 2024-09-13 05:43:58.72541915 +0000 UTC m=+5.684625699 container died 260b18297eb59de6e30c12a948bb8bd6cdb0ac11bf219da257eea037f29c1b40 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T05:43:59.244 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:59 smithi110 podman[69032]: 2024-09-13 05:43:59.16252973 +0000 UTC m=+6.121736269 container cleanup 260b18297eb59de6e30c12a948bb8bd6cdb0ac11bf219da257eea037f29c1b40 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T05:43:59.244 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:59 smithi110 bash[69032]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6 2024-09-13T05:43:59.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:59 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:59.196+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:43:59.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:59 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:59.196+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:43:59.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:59 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:59.196+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:43:59.244 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:43:59 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:43:59.196+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:43:59.744 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:59 smithi110 podman[69064]: 2024-09-13 05:43:59.364461881 +0000 UTC m=+0.633538269 container remove 260b18297eb59de6e30c12a948bb8bd6cdb0ac11bf219da257eea037f29c1b40 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:44:00.034 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:59 smithi110 podman[69153]: 2024-09-13 05:43:59.75681133 +0000 UTC m=+0.099014502 container create 4660e2a9c8b8736dc12077815abded1475afc0d7bf26304d044a6853873e7e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T05:44:00.034 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:59 smithi110 podman[69153]: 2024-09-13 05:43:59.673609948 +0000 UTC m=+0.015813078 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:44:00.034 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:59 smithi110 podman[69153]: 2024-09-13 05:43:59.959124465 +0000 UTC m=+0.301327594 container init 4660e2a9c8b8736dc12077815abded1475afc0d7bf26304d044a6853873e7e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:44:00.034 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:43:59 smithi110 podman[69153]: 2024-09-13 05:43:59.96239126 +0000 UTC m=+0.304594398 container start 4660e2a9c8b8736dc12077815abded1475afc0d7bf26304d044a6853873e7e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T05:44:00.034 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:44:00 smithi110 podman[69153]: 2024-09-13 05:44:00.0341567 +0000 UTC m=+0.376359854 container attach 4660e2a9c8b8736dc12077815abded1475afc0d7bf26304d044a6853873e7e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T05:44:00.429 INFO:journalctl@ceph.osd.6.smithi110.stdout:Sep 13 05:44:00 smithi110 podman[69153]: 2024-09-13 05:44:00.147534032 +0000 UTC m=+0.489737162 container died 4660e2a9c8b8736dc12077815abded1475afc0d7bf26304d044a6853873e7e25 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-13T05:44:00.430 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:00 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:00.191+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:44:00.430 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:00 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:00.191+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:44:00.430 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:00 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:00.191+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:44:00.430 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:00 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:00.191+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:44:00.821 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T05:44:00.860 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:44:00.860 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T05:44:00.860 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T05:44:00.861 DEBUG:teuthology.orchestra.run.smithi110:> sudo systemctl stop ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.7 2024-09-13T05:44:01.183 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:00 smithi110 systemd[1]: Stopping Ceph osd.7 for 9f95c28e-718f-11ef-bceb-c7b262605968... 2024-09-13T05:44:01.183 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:01.181+0000 7f27ede61640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T05:44:01.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:01.181+0000 7f27ede61640 -1 osd.7 155 *** Got signal Terminated *** 2024-09-13T05:44:01.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:01.181+0000 7f27ede61640 -1 osd.7 155 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T05:44:01.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:01.199+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:44:01.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:01.199+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:44:01.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:01.199+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:44:01.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:01 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:01.199+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:44:02.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:02 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:02.223+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:44:02.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:02 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:02.223+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:44:02.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:02 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:02.223+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:44:02.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:02 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:02.223+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:44:03.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:03.194+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:44:03.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:03.194+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:44:03.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:03.194+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:44:03.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:03.194+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:44:03.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:03 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:03.194+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.110:6804 osd.4 since back 2024-09-13T05:43:41.011411+0000 front 2024-09-13T05:43:41.011426+0000 (oldest deadline 2024-09-13T05:44:02.711428+0000) 2024-09-13T05:44:04.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:04 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:04.233+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:44:04.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:04 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:04.233+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:44:04.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:04 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:04.233+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:44:04.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:04 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:04.233+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:44:04.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:04 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:04.233+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.110:6804 osd.4 since back 2024-09-13T05:43:41.011411+0000 front 2024-09-13T05:43:41.011426+0000 (oldest deadline 2024-09-13T05:44:02.711428+0000) 2024-09-13T05:44:05.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:05 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:05.236+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6806 osd.0 since back 2024-09-13T05:43:07.807224+0000 front 2024-09-13T05:43:07.807362+0000 (oldest deadline 2024-09-13T05:43:31.807770+0000) 2024-09-13T05:44:05.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:05 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:05.236+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6814 osd.1 since back 2024-09-13T05:43:17.708822+0000 front 2024-09-13T05:43:17.708749+0000 (oldest deadline 2024-09-13T05:43:42.408649+0000) 2024-09-13T05:44:05.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:05 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:05.236+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6822 osd.2 since back 2024-09-13T05:43:25.810044+0000 front 2024-09-13T05:43:25.809972+0000 (oldest deadline 2024-09-13T05:43:50.509948+0000) 2024-09-13T05:44:05.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:05 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:05.236+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.53:6830 osd.3 since back 2024-09-13T05:43:30.510495+0000 front 2024-09-13T05:43:30.510540+0000 (oldest deadline 2024-09-13T05:43:56.410345+0000) 2024-09-13T05:44:05.494 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:05 smithi110 ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7[59402]: 2024-09-13T05:44:05.236+0000 7f27e646e640 -1 osd.7 155 heartbeat_check: no reply from 172.21.15.110:6804 osd.4 since back 2024-09-13T05:43:41.011411+0000 front 2024-09-13T05:43:41.011426+0000 (oldest deadline 2024-09-13T05:44:02.711428+0000) 2024-09-13T05:44:07.042 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:06 smithi110 podman[69257]: 2024-09-13 05:44:06.781320129 +0000 UTC m=+5.700046467 container died 9afa18d580fe142d289b94e76b987fc902c886767fb9c5f8c14f1fd327ba760b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T05:44:07.392 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:07 smithi110 podman[69257]: 2024-09-13 05:44:07.132708139 +0000 UTC m=+6.051434475 container cleanup 9afa18d580fe142d289b94e76b987fc902c886767fb9c5f8c14f1fd327ba760b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-09-13T05:44:07.393 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:07 smithi110 bash[69257]: ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7 2024-09-13T05:44:07.744 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:07 smithi110 podman[69286]: 2024-09-13 05:44:07.392813538 +0000 UTC m=+0.611765114 container remove 9afa18d580fe142d289b94e76b987fc902c886767fb9c5f8c14f1fd327ba760b (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T05:44:08.019 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:07 smithi110 podman[69373]: 2024-09-13 05:44:07.647828219 +0000 UTC m=+0.016061137 image pull 0ce38589d192f00a67c8f9b0cb5ac7ff2f64a4d81462c062375b6ecefcaf776f quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff 2024-09-13T05:44:08.019 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:07 smithi110 podman[69373]: 2024-09-13 05:44:07.745848936 +0000 UTC m=+0.114081844 container create 58d7c82eafb07b9ee208cf7ba51fa1cbadc8504615b8bd92c69c3c7f2b4bfb35 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T05:44:08.019 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:07 smithi110 podman[69373]: 2024-09-13 05:44:07.943670139 +0000 UTC m=+0.311903048 container init 58d7c82eafb07b9ee208cf7ba51fa1cbadc8504615b8bd92c69c3c7f2b4bfb35 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T05:44:08.020 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:07 smithi110 podman[69373]: 2024-09-13 05:44:07.946853909 +0000 UTC m=+0.315086827 container start 58d7c82eafb07b9ee208cf7ba51fa1cbadc8504615b8bd92c69c3c7f2b4bfb35 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-13T05:44:08.020 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:08 smithi110 podman[69373]: 2024-09-13 05:44:08.019343767 +0000 UTC m=+0.387576678 container attach 58d7c82eafb07b9ee208cf7ba51fa1cbadc8504615b8bd92c69c3c7f2b4bfb35 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731) 2024-09-13T05:44:08.433 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:08 smithi110 podman[69373]: 2024-09-13 05:44:08.132030818 +0000 UTC m=+0.500263736 container died 58d7c82eafb07b9ee208cf7ba51fa1cbadc8504615b8bd92c69c3c7f2b4bfb35 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T05:44:08.698 INFO:journalctl@ceph.osd.7.smithi110.stdout:Sep 13 05:44:08 smithi110 podman[69396]: 2024-09-13 05:44:08.678878087 +0000 UTC m=+0.537759151 container remove 58d7c82eafb07b9ee208cf7ba51fa1cbadc8504615b8bd92c69c3c7f2b4bfb35 (image=quay.ceph.io/ceph-ci/ceph@sha256:931e75f8a8ba7bd669ee47053a4500e7d74632603919af8ba169df149fde6cff, name=ceph-9f95c28e-718f-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-12-1245-9d608eb) 2024-09-13T05:44:08.703 DEBUG:teuthology.orchestra.run.smithi110:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9f95c28e-718f-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T05:44:08.741 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T05:44:08.742 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T05:44:08.742 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T05:47:04.405 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T05:47:24.428 DEBUG:teuthology.orchestra.run.smithi053:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T05:47:24.463 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T05:47:24.493 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T05:47:24.496 DEBUG:teuthology.misc:Transferring archived files from smithi053:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116/remote/smithi053/crash 2024-09-13T05:47:24.497 DEBUG:teuthology.orchestra.run.smithi053:> sudo tar c -f - -C /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/crash -- . 2024-09-13T05:47:24.542 INFO:teuthology.orchestra.run.smithi053.stderr:tar: /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T05:47:24.543 INFO:teuthology.orchestra.run.smithi053.stderr:tar: Error is not recoverable: exiting now 2024-09-13T05:47:24.545 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116/remote/smithi110/crash 2024-09-13T05:47:24.546 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/crash -- . 2024-09-13T05:47:24.580 INFO:teuthology.orchestra.run.smithi110.stderr:tar: /var/lib/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T05:47:24.580 INFO:teuthology.orchestra.run.smithi110.stderr:tar: Error is not recoverable: exiting now 2024-09-13T05:47:24.581 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T05:47:24.581 DEBUG:teuthology.orchestra.run.smithi053:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-13T05:47:24.614 INFO:tasks.cephadm:Compressing logs... 2024-09-13T05:47:24.614 DEBUG:teuthology.orchestra.run.smithi053:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T05:47:24.656 DEBUG:teuthology.orchestra.run.smithi110:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T05:47:24.680 INFO:teuthology.orchestra.run.smithi053.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T05:47:24.681 INFO:teuthology.orchestra.run.smithi053.stderr:: No such file or directory 2024-09-13T05:47:24.681 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mgr.smithi053.ygzjut.log 2024-09-13T05:47:24.681 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T05:47:24.681 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mgr.smithi053.ygzjut.log: -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi053.rzvrng.log 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-volume.log: /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi053.rzvrng.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mon.smithi053.log 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr: 89.1% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi053.rzvrng.log.gz 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mon.smithi053.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.log 2024-09-13T05:47:24.686 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.2.log: 85.7% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mon.smithi110.log 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mon.smithi110.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T05:47:24.689 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.4.log: /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mgr.smithi110.rivldx.log 2024-09-13T05:47:24.690 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T05:47:24.690 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mgr.smithi110.rivldx.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T05:47:24.690 INFO:teuthology.orchestra.run.smithi110.stderr: 85.7% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T05:47:24.690 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.log 2024-09-13T05:47:24.690 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T05:47:24.690 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi110.akbygi.log 2024-09-13T05:47:24.691 INFO:teuthology.orchestra.run.smithi053.stderr: 91.5% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T05:47:24.691 INFO:teuthology.orchestra.run.smithi053.stderr:gzip -5 --verbose -- /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T05:47:24.691 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T05:47:24.692 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.log: 87.1% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T05:47:24.692 INFO:teuthology.orchestra.run.smithi110.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T05:47:24.693 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi110.akbygi.log: 87.7% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi110.akbygi.log.gz 2024-09-13T05:47:24.693 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.3.log: 87.1% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T05:47:24.696 INFO:teuthology.orchestra.run.smithi110.stderr:/var/log/ceph/cephadm.log: 91.3% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T05:47:24.697 INFO:teuthology.orchestra.run.smithi053.stderr: 91.1% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T05:47:24.700 INFO:teuthology.orchestra.run.smithi110.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T05:47:24.745 INFO:teuthology.orchestra.run.smithi053.stderr:/var/log/ceph/cephadm.log: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T05:47:24.753 INFO:teuthology.orchestra.run.smithi110.stderr: 89.7% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mgr.smithi110.rivldx.log.gz 2024-09-13T05:47:24.949 INFO:teuthology.orchestra.run.smithi053.stderr: 88.7% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mgr.smithi053.ygzjut.log.gz 2024-09-13T05:47:25.005 INFO:teuthology.orchestra.run.smithi110.stderr: 92.6% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mon.smithi110.log.gz 2024-09-13T05:47:26.068 INFO:teuthology.orchestra.run.smithi053.stderr: 91.7% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-mon.smithi053.log.gz 2024-09-13T05:47:35.812 INFO:teuthology.orchestra.run.smithi053.stderr: 94.4% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T05:47:37.369 INFO:teuthology.orchestra.run.smithi110.stderr: 93.9% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T05:47:38.691 INFO:teuthology.orchestra.run.smithi110.stderr: 94.2% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T05:47:39.925 INFO:teuthology.orchestra.run.smithi053.stderr: 94.0% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T05:47:41.640 INFO:teuthology.orchestra.run.smithi053.stderr: 94.0% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T05:47:42.141 INFO:teuthology.orchestra.run.smithi053.stderr: 94.2% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T05:47:42.143 INFO:teuthology.orchestra.run.smithi053.stderr: 2024-09-13T05:47:42.143 INFO:teuthology.orchestra.run.smithi053.stderr:real 0m17.473s 2024-09-13T05:47:42.143 INFO:teuthology.orchestra.run.smithi053.stderr:user 1m0.174s 2024-09-13T05:47:42.143 INFO:teuthology.orchestra.run.smithi053.stderr:sys 0m2.241s 2024-09-13T05:47:42.170 INFO:teuthology.orchestra.run.smithi110.stderr: 94.2% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T05:47:47.080 INFO:teuthology.orchestra.run.smithi110.stderr: 94.1% -- replaced with /var/log/ceph/9f95c28e-718f-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T05:47:47.082 INFO:teuthology.orchestra.run.smithi110.stderr: 2024-09-13T05:47:47.082 INFO:teuthology.orchestra.run.smithi110.stderr:real 0m22.411s 2024-09-13T05:47:47.082 INFO:teuthology.orchestra.run.smithi110.stderr:user 1m4.332s 2024-09-13T05:47:47.082 INFO:teuthology.orchestra.run.smithi110.stderr:sys 0m2.567s 2024-09-13T05:47:47.082 INFO:tasks.cephadm:Archiving logs... 2024-09-13T05:47:47.083 DEBUG:teuthology.misc:Transferring archived files from smithi053:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116/remote/smithi053/log 2024-09-13T05:47:47.084 DEBUG:teuthology.orchestra.run.smithi053:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T05:47:55.726 DEBUG:teuthology.misc:Transferring archived files from smithi110:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116/remote/smithi110/log 2024-09-13T05:47:55.728 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T05:48:04.705 INFO:tasks.cephadm:Removing cluster... 2024-09-13T05:48:04.705 DEBUG:teuthology.orchestra.run.smithi053:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 --force 2024-09-13T05:48:05.069 DEBUG:teuthology.orchestra.run.smithi110:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9f95c28e-718f-11ef-bceb-c7b262605968 --force 2024-09-13T05:48:05.430 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T05:48:05.430 DEBUG:teuthology.orchestra.run.smithi053:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T05:48:05.448 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T05:48:05.467 INFO:tasks.cephadm:Teardown complete 2024-09-13T05:48:05.467 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T05:48:05.478 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T05:48:05.478 DEBUG:teuthology.orchestra.run.smithi053:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T05:48:05.491 DEBUG:teuthology.orchestra.run.smithi110:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T05:48:05.503 INFO:teuthology.orchestra.run.smithi053.stderr:bash: line 1: ntpq: command not found 2024-09-13T05:48:05.508 INFO:teuthology.orchestra.run.smithi053.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T05:48:05.508 INFO:teuthology.orchestra.run.smithi053.stdout:=============================================================================== 2024-09-13T05:48:05.509 INFO:teuthology.orchestra.run.smithi053.stdout:^* hv01.front.sepia.ceph.com 3 6 377 16 +496us[ +566us] +/- 46ms 2024-09-13T05:48:05.509 INFO:teuthology.orchestra.run.smithi053.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 19 -1034us[ -965us] +/- 61ms 2024-09-13T05:48:05.509 INFO:teuthology.orchestra.run.smithi053.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 17 +603us[ +672us] +/- 89ms 2024-09-13T05:48:05.509 INFO:teuthology.orchestra.run.smithi053.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:48:05.522 INFO:teuthology.orchestra.run.smithi110.stderr:bash: line 1: ntpq: command not found 2024-09-13T05:48:05.526 INFO:teuthology.orchestra.run.smithi110.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T05:48:05.527 INFO:teuthology.orchestra.run.smithi110.stdout:=============================================================================== 2024-09-13T05:48:05.527 INFO:teuthology.orchestra.run.smithi110.stdout:^* hv01.front.sepia.ceph.com 3 6 377 17 +500us[ +547us] +/- 46ms 2024-09-13T05:48:05.527 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 18 -1043us[ -996us] +/- 61ms 2024-09-13T05:48:05.527 INFO:teuthology.orchestra.run.smithi110.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 19 +584us[ +631us] +/- 89ms 2024-09-13T05:48:05.527 INFO:teuthology.orchestra.run.smithi110.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T05:48:05.527 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T05:48:05.537 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T05:48:05.538 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T05:48:05.548 DEBUG:teuthology.orchestra.run.smithi053:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T05:48:05.631 DEBUG:teuthology.orchestra.run.smithi110:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T05:48:05.702 DEBUG:teuthology.orchestra.run.smithi053:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T05:48:05.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:48:05.732 DEBUG:teuthology.orchestra.run.smithi110:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T05:48:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:48:05.759 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T05:48:05.769 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T05:48:05.778 INFO:teuthology.task.internal:Duration was 1999.985457 seconds 2024-09-13T05:48:05.779 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T05:48:05.788 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T05:48:05.788 DEBUG:teuthology.orchestra.run.smithi053:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T05:48:05.790 DEBUG:teuthology.orchestra.run.smithi110:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T05:48:05.829 INFO:teuthology.orchestra.run.smithi053.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T05:48:05.837 INFO:teuthology.orchestra.run.smithi110.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T05:48:06.281 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T05:48:06.281 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi053.front.sepia.ceph.com 2024-09-13T05:48:06.282 DEBUG:teuthology.orchestra.run.smithi053:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T05:48:06.304 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi110.front.sepia.ceph.com 2024-09-13T05:48:06.305 DEBUG:teuthology.orchestra.run.smithi110:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T05:48:06.349 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T05:48:06.349 DEBUG:teuthology.orchestra.run.smithi053:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T05:48:06.351 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T05:48:06.416 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T05:48:06.416 DEBUG:teuthology.orchestra.run.smithi053:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T05:48:06.420 DEBUG:teuthology.orchestra.run.smithi110:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T05:48:20.675 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T05:48:20.697 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T05:48:20.698 DEBUG:teuthology.orchestra.run.smithi053:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T05:48:20.725 DEBUG:teuthology.orchestra.run.smithi110:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T05:48:20.755 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T05:48:20.767 DEBUG:teuthology.orchestra.run.smithi053:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T05:48:20.770 DEBUG:teuthology.orchestra.run.smithi110:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T05:48:20.792 INFO:teuthology.orchestra.run.smithi053.stdout:kernel.core_pattern = core 2024-09-13T05:48:20.823 INFO:teuthology.orchestra.run.smithi110.stdout:kernel.core_pattern = core 2024-09-13T05:48:20.847 DEBUG:teuthology.orchestra.run.smithi053:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T05:48:20.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:48:20.870 DEBUG:teuthology.orchestra.run.smithi110:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T05:48:20.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T05:48:20.903 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T05:48:20.913 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T05:48:20.914 DEBUG:teuthology.misc:Transferring archived files from smithi053:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116/remote/smithi053 2024-09-13T05:48:20.914 DEBUG:teuthology.orchestra.run.smithi053:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T05:48:20.976 DEBUG:teuthology.misc:Transferring archived files from smithi110:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-13_05:00:55-orch:cephadm-wip-guits-main-2024-09-12-1245-distro-default-smithi/7903116/remote/smithi110 2024-09-13T05:48:20.977 DEBUG:teuthology.orchestra.run.smithi110:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T05:48:21.020 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T05:48:21.020 DEBUG:teuthology.orchestra.run.smithi053:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T05:48:21.023 DEBUG:teuthology.orchestra.run.smithi110:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T05:48:21.070 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T05:48:21.082 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T05:48:21.082 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T05:48:21.093 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T05:48:21.093 DEBUG:teuthology.orchestra.run.smithi053:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T05:48:21.096 DEBUG:teuthology.orchestra.run.smithi110:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T05:48:21.109 INFO:teuthology.orchestra.run.smithi053.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 05:48 /home/ubuntu/cephtest 2024-09-13T05:48:21.126 INFO:teuthology.orchestra.run.smithi110.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 05:48 /home/ubuntu/cephtest 2024-09-13T05:48:21.127 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T05:48:21.137 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T05:48:21.160 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1999.9854567050934 owner: scheduled_gabrioux@teuthology success: true 2024-09-13T05:48:21.160 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T05:48:21.262 INFO:teuthology.run:pass